Spaces:
Running
Running
Merge feature/naming-convention-fix into feature/chat-interface
Browse files- CLAUDE.md +1 -1
- cdw_connector.py +36 -10
- demo_logger.py +87 -29
- demo_prep.py +138 -60
- supabase_client.py +1 -0
- tests/debug_population_error.py +0 -245
- tests/debug_schema_utils.py +0 -106
- tests/test_connection_success.py +0 -123
- tests/test_core_success.py +0 -150
- tests/test_deployment_flow.py +0 -54
- tests/test_end_to_end_demo.py +0 -174
- tests/test_framework.py +0 -447
CLAUDE.md
CHANGED
|
@@ -213,4 +213,4 @@ This software is curretnly stored in my repo but will be open sourced and move t
|
|
| 213 |
---
|
| 214 |
|
| 215 |
*Last Updated: December 10, 2025*
|
| 216 |
-
*This is a living document - update as you learn more about the project*
|
|
|
|
| 213 |
---
|
| 214 |
|
| 215 |
*Last Updated: December 10, 2025*
|
| 216 |
+
*This is a living document - update as you learn more about the project*
|
cdw_connector.py
CHANGED
|
@@ -72,19 +72,40 @@ class SnowflakeDeployer:
|
|
| 72 |
except Exception as e:
|
| 73 |
print(f"Error closing connection: {str(e)}")
|
| 74 |
|
| 75 |
-
def create_demo_schema_and_deploy(self, company_name: str, use_case: str, ddl_statements: str) -> Tuple[bool, Optional[str], str]:
|
| 76 |
-
"""Create timestamped schema and deploy DDL statements
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
if not self.connection:
|
| 78 |
success, message = self.connect()
|
| 79 |
if not success:
|
| 80 |
return False, None, f"Connection failed: {message}"
|
| 81 |
-
|
| 82 |
try:
|
| 83 |
-
# Generate schema name
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
cursor = self.connection.cursor()
|
| 90 |
|
|
@@ -127,15 +148,20 @@ class SnowflakeDeployer:
|
|
| 127 |
# Check autocommit setting
|
| 128 |
print(f"🔍 DEBUG: Connection autocommit: {self.connection.autocommit}")
|
| 129 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
# VERIFY the schema actually exists
|
| 131 |
print(f"🔍 DEBUG: Verifying schema exists...")
|
| 132 |
cursor.execute(f'SELECT CURRENT_SCHEMA()')
|
| 133 |
current_schema = cursor.fetchone()[0]
|
| 134 |
print(f"🔍 DEBUG: Current schema: {current_schema}")
|
| 135 |
print(f"🔍 DEBUG: Schema verification completed - using current schema context")
|
| 136 |
-
|
| 137 |
cursor.close()
|
| 138 |
-
|
| 139 |
success_message = f"Schema '{schema_name}' created successfully with {executed_count} tables"
|
| 140 |
return True, schema_name, success_message
|
| 141 |
|
|
|
|
| 72 |
except Exception as e:
|
| 73 |
print(f"Error closing connection: {str(e)}")
|
| 74 |
|
| 75 |
+
def create_demo_schema_and_deploy(self, company_name: str, use_case: str, ddl_statements: str, naming_prefix: str = "") -> Tuple[bool, Optional[str], str]:
|
| 76 |
+
"""Create timestamped schema and deploy DDL statements
|
| 77 |
+
|
| 78 |
+
Args:
|
| 79 |
+
naming_prefix: Optional custom prefix for schema name
|
| 80 |
+
Format: {prefix}_{day+ms}_scm or just {day+ms}_scm if blank
|
| 81 |
+
"""
|
| 82 |
if not self.connection:
|
| 83 |
success, message = self.connect()
|
| 84 |
if not success:
|
| 85 |
return False, None, f"Connection failed: {message}"
|
| 86 |
+
|
| 87 |
try:
|
| 88 |
+
# Generate schema name using same convention as ThoughtSpot objects
|
| 89 |
+
import re
|
| 90 |
+
from datetime import datetime as dt
|
| 91 |
+
now = dt.now()
|
| 92 |
+
|
| 93 |
+
# Clean prefix (uppercase, alphanumeric only) - can be empty string
|
| 94 |
+
if naming_prefix:
|
| 95 |
+
prefix_clean = re.sub(r'[^a-zA-Z0-9]', '', naming_prefix.upper())
|
| 96 |
+
else:
|
| 97 |
+
prefix_clean = ""
|
| 98 |
+
|
| 99 |
+
# Short unique timestamp: day (2 digits) + microseconds (6 digits)
|
| 100 |
+
day = now.strftime('%d')
|
| 101 |
+
microsec = now.strftime('%f') # 6 digits
|
| 102 |
+
short_timestamp = f"{day}{microsec}"
|
| 103 |
+
|
| 104 |
+
# Base format: PREFIX_DDMICROS_scm (prefix can be empty)
|
| 105 |
+
if prefix_clean:
|
| 106 |
+
schema_name = f"{prefix_clean}_{short_timestamp}_scm"
|
| 107 |
+
else:
|
| 108 |
+
schema_name = f"{short_timestamp}_scm"
|
| 109 |
|
| 110 |
cursor = self.connection.cursor()
|
| 111 |
|
|
|
|
| 148 |
# Check autocommit setting
|
| 149 |
print(f"🔍 DEBUG: Connection autocommit: {self.connection.autocommit}")
|
| 150 |
|
| 151 |
+
# COMMIT the transaction
|
| 152 |
+
print(f"🔍 DEBUG: Committing transaction...")
|
| 153 |
+
self.connection.commit()
|
| 154 |
+
print(f"✅ Transaction committed - {executed_count} tables created")
|
| 155 |
+
|
| 156 |
# VERIFY the schema actually exists
|
| 157 |
print(f"🔍 DEBUG: Verifying schema exists...")
|
| 158 |
cursor.execute(f'SELECT CURRENT_SCHEMA()')
|
| 159 |
current_schema = cursor.fetchone()[0]
|
| 160 |
print(f"🔍 DEBUG: Current schema: {current_schema}")
|
| 161 |
print(f"🔍 DEBUG: Schema verification completed - using current schema context")
|
| 162 |
+
|
| 163 |
cursor.close()
|
| 164 |
+
|
| 165 |
success_message = f"Schema '{schema_name}' created successfully with {executed_count} tables"
|
| 166 |
return True, schema_name, success_message
|
| 167 |
|
demo_logger.py
CHANGED
|
@@ -25,17 +25,24 @@ load_dotenv()
|
|
| 25 |
class DemoLogger:
|
| 26 |
"""Comprehensive logger for demo creation process"""
|
| 27 |
|
| 28 |
-
def __init__(self, session_id: str = None, user_email: str = None):
|
| 29 |
"""
|
| 30 |
Initialize logger with session ID and optional user email
|
| 31 |
|
| 32 |
Args:
|
| 33 |
session_id: Unique ID for this demo creation session
|
| 34 |
user_email: User's email for tracking
|
|
|
|
| 35 |
"""
|
| 36 |
self.session_id = session_id or datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 37 |
self.user_email = user_email or "anonymous"
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
# Create logs directory
|
| 40 |
self.log_dir = Path("logs") / "demo_sessions" / self.session_id
|
| 41 |
self.log_dir.mkdir(parents=True, exist_ok=True)
|
|
@@ -64,7 +71,14 @@ class DemoLogger:
|
|
| 64 |
"status": "started"
|
| 65 |
}
|
| 66 |
|
| 67 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
|
| 69 |
def setup_logger(self):
|
| 70 |
"""Set up Python logger"""
|
|
@@ -100,47 +114,79 @@ class DemoLogger:
|
|
| 100 |
self.session_data["llm_model"] = llm_model
|
| 101 |
self.session_data["settings"].update(kwargs)
|
| 102 |
|
| 103 |
-
self.logger.info(
|
| 104 |
-
self.logger.info(
|
| 105 |
-
self.logger.info(
|
| 106 |
-
self.logger.info(f"
|
| 107 |
-
self.logger.info(f"
|
| 108 |
-
self.logger.info(f"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
|
| 110 |
self.save_session_data()
|
| 111 |
|
| 112 |
def log_ddl(self, ddl_content: str, schema_name: str = None):
|
| 113 |
-
"""Log the generated DDL"""
|
| 114 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
self.session_data["schema_name"] = schema_name
|
| 116 |
|
| 117 |
-
#
|
| 118 |
ddl_file = self.log_dir / "generated_ddl.sql"
|
| 119 |
with open(ddl_file, 'w') as f:
|
| 120 |
f.write(ddl_content)
|
| 121 |
|
| 122 |
# Log summary
|
| 123 |
tables = ddl_content.count("CREATE TABLE")
|
| 124 |
-
self.logger.info(
|
| 125 |
-
self.logger.info(f"DDL
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
|
| 131 |
self.save_session_data()
|
| 132 |
|
| 133 |
def log_population_code(self, code: str):
|
| 134 |
-
"""Log the generated population code"""
|
| 135 |
-
|
|
|
|
| 136 |
|
| 137 |
-
#
|
| 138 |
code_file = self.log_dir / "population_code.py"
|
| 139 |
with open(code_file, 'w') as f:
|
| 140 |
f.write(code)
|
| 141 |
|
| 142 |
-
self.logger.info(
|
| 143 |
-
self.logger.info(f"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
|
| 145 |
self.save_session_data()
|
| 146 |
|
|
@@ -148,9 +194,16 @@ class DemoLogger:
|
|
| 148 |
"""Log a discovered outlier"""
|
| 149 |
self.session_data["outliers"].append(outlier)
|
| 150 |
|
| 151 |
-
self.logger.info(f"Outlier discovered: {outlier.get('title', 'Unknown')}")
|
| 152 |
-
self.
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
|
| 155 |
self.save_session_data()
|
| 156 |
|
|
@@ -190,13 +243,14 @@ class DemoLogger:
|
|
| 190 |
self.save_session_data()
|
| 191 |
|
| 192 |
def log_stage_completion(self, stage: str, success: bool,
|
| 193 |
-
duration_seconds: float = None):
|
| 194 |
"""Log completion of a demo creation stage"""
|
| 195 |
stage_data = {
|
| 196 |
"stage": stage,
|
| 197 |
"success": success,
|
| 198 |
"duration": duration_seconds,
|
| 199 |
-
"timestamp": datetime.now().isoformat()
|
|
|
|
| 200 |
}
|
| 201 |
|
| 202 |
if "stages" not in self.session_data:
|
|
@@ -205,9 +259,13 @@ class DemoLogger:
|
|
| 205 |
self.session_data["stages"].append(stage_data)
|
| 206 |
|
| 207 |
status = "✅ SUCCESS" if success else "❌ FAILED"
|
| 208 |
-
self.logger.info(
|
|
|
|
| 209 |
if duration_seconds:
|
| 210 |
-
self.logger.info(f"
|
|
|
|
|
|
|
|
|
|
| 211 |
|
| 212 |
self.save_session_data()
|
| 213 |
|
|
|
|
| 25 |
class DemoLogger:
|
| 26 |
"""Comprehensive logger for demo creation process"""
|
| 27 |
|
| 28 |
+
def __init__(self, session_id: str = None, user_email: str = None, debug_mode: bool = None):
|
| 29 |
"""
|
| 30 |
Initialize logger with session ID and optional user email
|
| 31 |
|
| 32 |
Args:
|
| 33 |
session_id: Unique ID for this demo creation session
|
| 34 |
user_email: User's email for tracking
|
| 35 |
+
debug_mode: Enable debug mode (saves DDL, population code, outliers)
|
| 36 |
"""
|
| 37 |
self.session_id = session_id or datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 38 |
self.user_email = user_email or "anonymous"
|
| 39 |
|
| 40 |
+
# Check for DEBUG environment variable or parameter
|
| 41 |
+
if debug_mode is None:
|
| 42 |
+
self.debug_mode = os.getenv('DEBUG', 'false').lower() in ['true', '1', 'yes']
|
| 43 |
+
else:
|
| 44 |
+
self.debug_mode = debug_mode
|
| 45 |
+
|
| 46 |
# Create logs directory
|
| 47 |
self.log_dir = Path("logs") / "demo_sessions" / self.session_id
|
| 48 |
self.log_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
| 71 |
"status": "started"
|
| 72 |
}
|
| 73 |
|
| 74 |
+
# Log session start with timestamp
|
| 75 |
+
self.logger.info("="*80)
|
| 76 |
+
self.logger.info(f"🚀 Demo Session Started")
|
| 77 |
+
self.logger.info(f" Session ID: {self.session_id}")
|
| 78 |
+
self.logger.info(f" User: {self.user_email}")
|
| 79 |
+
self.logger.info(f" Timestamp: {self.session_data['start_time']}")
|
| 80 |
+
self.logger.info(f" Debug Mode: {'ON' if self.debug_mode else 'OFF'}")
|
| 81 |
+
self.logger.info("="*80)
|
| 82 |
|
| 83 |
def setup_logger(self):
|
| 84 |
"""Set up Python logger"""
|
|
|
|
| 114 |
self.session_data["llm_model"] = llm_model
|
| 115 |
self.session_data["settings"].update(kwargs)
|
| 116 |
|
| 117 |
+
self.logger.info("="*80)
|
| 118 |
+
self.logger.info("📋 Demo Configuration")
|
| 119 |
+
self.logger.info("="*80)
|
| 120 |
+
self.logger.info(f"🎯 MAIN SETTINGS:")
|
| 121 |
+
self.logger.info(f" Company URL: {company_url}")
|
| 122 |
+
self.logger.info(f" Use Case: {use_case}")
|
| 123 |
+
self.logger.info(f" LLM Model: {llm_model}")
|
| 124 |
+
self.logger.info(f"")
|
| 125 |
+
self.logger.info(f"📊 Additional Info:")
|
| 126 |
+
self.logger.info(f" Company: {company}")
|
| 127 |
+
self.logger.info(f" Industry: {industry}")
|
| 128 |
+
|
| 129 |
+
if kwargs:
|
| 130 |
+
self.logger.info(f"")
|
| 131 |
+
self.logger.info(f"⚙️ Other Settings:")
|
| 132 |
+
for key, value in kwargs.items():
|
| 133 |
+
self.logger.info(f" {key}: {value}")
|
| 134 |
+
|
| 135 |
+
self.logger.info("="*80)
|
| 136 |
|
| 137 |
self.save_session_data()
|
| 138 |
|
| 139 |
def log_ddl(self, ddl_content: str, schema_name: str = None):
|
| 140 |
+
"""Log the generated DDL - ALWAYS saves to file"""
|
| 141 |
+
# Always save full DDL if debug mode
|
| 142 |
+
if self.debug_mode:
|
| 143 |
+
self.session_data["ddl"] = ddl_content
|
| 144 |
+
else:
|
| 145 |
+
self.session_data["ddl"] = ddl_content[:1000] + "..." if len(ddl_content) > 1000 else ddl_content
|
| 146 |
+
|
| 147 |
self.session_data["schema_name"] = schema_name
|
| 148 |
|
| 149 |
+
# ALWAYS save DDL to separate file
|
| 150 |
ddl_file = self.log_dir / "generated_ddl.sql"
|
| 151 |
with open(ddl_file, 'w') as f:
|
| 152 |
f.write(ddl_content)
|
| 153 |
|
| 154 |
# Log summary
|
| 155 |
tables = ddl_content.count("CREATE TABLE")
|
| 156 |
+
self.logger.info("="*80)
|
| 157 |
+
self.logger.info(f"📝 DDL Generated")
|
| 158 |
+
self.logger.info("="*80)
|
| 159 |
+
self.logger.info(f" Tables: {tables}")
|
| 160 |
+
self.logger.info(f" Size: {len(ddl_content)} characters")
|
| 161 |
+
self.logger.info(f" Schema: {schema_name or 'N/A'}")
|
| 162 |
+
self.logger.info(f" Saved to: {ddl_file}")
|
| 163 |
+
|
| 164 |
+
# Check for SALESTRANSACTIONS or SALES_TRANSACTIONS table (known issue)
|
| 165 |
+
ddl_upper = ddl_content.upper()
|
| 166 |
+
if "SALESTRANSACTIONS" not in ddl_upper and "SALES_TRANSACTIONS" not in ddl_upper:
|
| 167 |
+
self.logger.warning("⚠️ SALESTRANSACTIONS table not found in DDL - this will cause deployment issues!")
|
| 168 |
+
self.log_warning("SALESTRANSACTIONS table not found in DDL")
|
| 169 |
+
|
| 170 |
+
self.logger.info("="*80)
|
| 171 |
|
| 172 |
self.save_session_data()
|
| 173 |
|
| 174 |
def log_population_code(self, code: str):
|
| 175 |
+
"""Log the generated population code - saves to file if DEBUG mode"""
|
| 176 |
+
# Always save summary
|
| 177 |
+
self.session_data["population_code"] = code[:5000] if not self.debug_mode else code
|
| 178 |
|
| 179 |
+
# ALWAYS save full code to file
|
| 180 |
code_file = self.log_dir / "population_code.py"
|
| 181 |
with open(code_file, 'w') as f:
|
| 182 |
f.write(code)
|
| 183 |
|
| 184 |
+
self.logger.info("="*80)
|
| 185 |
+
self.logger.info(f"🐍 Population Code Generated")
|
| 186 |
+
self.logger.info("="*80)
|
| 187 |
+
self.logger.info(f" Size: {len(code)} characters")
|
| 188 |
+
self.logger.info(f" Saved to: {code_file}")
|
| 189 |
+
self.logger.info("="*80)
|
| 190 |
|
| 191 |
self.save_session_data()
|
| 192 |
|
|
|
|
| 194 |
"""Log a discovered outlier"""
|
| 195 |
self.session_data["outliers"].append(outlier)
|
| 196 |
|
| 197 |
+
self.logger.info(f"💡 Outlier discovered: {outlier.get('title', 'Unknown')}")
|
| 198 |
+
if self.debug_mode:
|
| 199 |
+
self.logger.info(f" Insight: {outlier.get('insight', 'N/A')}")
|
| 200 |
+
self.logger.info(f" Impact: {outlier.get('impact', 'N/A')}")
|
| 201 |
+
|
| 202 |
+
# Save all outliers to file in DEBUG mode
|
| 203 |
+
if self.debug_mode:
|
| 204 |
+
outliers_file = self.log_dir / "outliers.json"
|
| 205 |
+
with open(outliers_file, 'w') as f:
|
| 206 |
+
json.dump(self.session_data["outliers"], f, indent=2)
|
| 207 |
|
| 208 |
self.save_session_data()
|
| 209 |
|
|
|
|
| 243 |
self.save_session_data()
|
| 244 |
|
| 245 |
def log_stage_completion(self, stage: str, success: bool,
|
| 246 |
+
duration_seconds: float = None, details: str = None):
|
| 247 |
"""Log completion of a demo creation stage"""
|
| 248 |
stage_data = {
|
| 249 |
"stage": stage,
|
| 250 |
"success": success,
|
| 251 |
"duration": duration_seconds,
|
| 252 |
+
"timestamp": datetime.now().isoformat(),
|
| 253 |
+
"details": details
|
| 254 |
}
|
| 255 |
|
| 256 |
if "stages" not in self.session_data:
|
|
|
|
| 259 |
self.session_data["stages"].append(stage_data)
|
| 260 |
|
| 261 |
status = "✅ SUCCESS" if success else "❌ FAILED"
|
| 262 |
+
self.logger.info("="*80)
|
| 263 |
+
self.logger.info(f"📍 Stage: {stage.upper()} - {status}")
|
| 264 |
if duration_seconds:
|
| 265 |
+
self.logger.info(f" Duration: {duration_seconds:.2f} seconds")
|
| 266 |
+
if details:
|
| 267 |
+
self.logger.info(f" Details: {details}")
|
| 268 |
+
self.logger.info("="*80)
|
| 269 |
|
| 270 |
self.save_session_data()
|
| 271 |
|
demo_prep.py
CHANGED
|
@@ -371,20 +371,31 @@ def execute_population_script(python_code, schema_name, skip_modifications=False
|
|
| 371 |
["conn_params.pop('schema', None) # Remove schema to avoid duplicate"]
|
| 372 |
)
|
| 373 |
else:
|
| 374 |
-
|
| 375 |
-
|
|
|
|
|
|
|
|
|
|
| 376 |
|
| 377 |
# Simple and safe schema replacement - just replace the placeholder
|
| 378 |
cleaned_code = cleaned_code.replace("os.getenv('SNOWFLAKE_SCHEMA')", f"'{schema_name}'")
|
| 379 |
cleaned_code = cleaned_code.replace('os.getenv("SNOWFLAKE_SCHEMA")', f'"{schema_name}"')
|
| 380 |
|
|
|
|
|
|
|
|
|
|
| 381 |
# FIX: Remove fake.unique() calls that cause "duplicated values after 1,000 iterations" error
|
| 382 |
cleaned_code = cleaned_code.replace("fake.unique.word()", "fake.word()")
|
| 383 |
cleaned_code = cleaned_code.replace("fake.unique.email()", "fake.email()")
|
| 384 |
cleaned_code = cleaned_code.replace("fake.unique.company()", "fake.company()")
|
| 385 |
|
| 386 |
# FIX: Truncate phone numbers to avoid extension overflow (e.g., '790-923-3730x07350')
|
| 387 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 388 |
|
| 389 |
# FIX: Convert SQLite-style ? placeholders to Snowflake-style %s placeholders
|
| 390 |
cleaned_code = re.sub(r'\bVALUES\s*\(\?', 'VALUES (%s', cleaned_code)
|
|
@@ -563,6 +574,7 @@ def execute_population_script(python_code, schema_name, skip_modifications=False
|
|
| 563 |
"os": os,
|
| 564 |
"random": random,
|
| 565 |
"datetime": datetime,
|
|
|
|
| 566 |
"get_snowflake_connection_params": get_snowflake_connection_params,
|
| 567 |
}
|
| 568 |
|
|
@@ -574,11 +586,24 @@ def execute_population_script(python_code, schema_name, skip_modifications=False
|
|
| 574 |
print(f"🔍 DEBUG: Cleaned code preview (first 500 chars):")
|
| 575 |
print(cleaned_code[:500])
|
| 576 |
print("...")
|
| 577 |
-
|
| 578 |
-
#
|
| 579 |
-
|
|
|
|
|
|
|
| 580 |
f.write(cleaned_code)
|
| 581 |
-
print(f"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 582 |
|
| 583 |
# First, validate the syntax before executing
|
| 584 |
try:
|
|
@@ -607,6 +632,24 @@ def execute_population_script(python_code, schema_name, skip_modifications=False
|
|
| 607 |
print("=" * 50)
|
| 608 |
print(f"❌ DATA POPULATION FAILED: {str(e)}")
|
| 609 |
print("=" * 50)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 610 |
return False, f"Population execution failed: {str(e)}"
|
| 611 |
|
| 612 |
def generate_deployment_summary(demo_builder_instance):
|
|
@@ -1018,7 +1061,7 @@ def create_demo_prep_interface():
|
|
| 1018 |
info="Primary AI model for all research and generation tasks"
|
| 1019 |
)
|
| 1020 |
default_company_url = gr.Textbox(
|
| 1021 |
-
value="
|
| 1022 |
label="Default Company URL",
|
| 1023 |
info="Pre-filled company URL for quick starts"
|
| 1024 |
)
|
|
@@ -1059,6 +1102,12 @@ def create_demo_prep_interface():
|
|
| 1059 |
label="Schema Prefix",
|
| 1060 |
info="Prefix for schema names (e.g., demo_20250930)"
|
| 1061 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1062 |
|
| 1063 |
with gr.TabItem("Admin"):
|
| 1064 |
ts_instance_url = gr.Textbox(
|
|
@@ -1262,7 +1311,7 @@ def create_demo_prep_interface():
|
|
| 1262 |
# Progressive workflow with Multi-LLM support
|
| 1263 |
def progressive_workflow_handler(
|
| 1264 |
url, use_case, llm_provider, demo_builder_instance, mode="expert", stop_before_deploy=False,
|
| 1265 |
-
temperature=0.3, max_tokens=4000
|
| 1266 |
):
|
| 1267 |
# Initialize DemoBuilder if none exists
|
| 1268 |
if demo_builder_instance is None:
|
|
@@ -1270,6 +1319,17 @@ def create_demo_prep_interface():
|
|
| 1270 |
return "Please enter a company URL", None, "Start Research", False
|
| 1271 |
demo_builder_instance = DemoBuilder(use_case, url.strip())
|
| 1272 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1273 |
# Initialize or continue with existing content
|
| 1274 |
if not hasattr(demo_builder_instance, 'accumulated_content'):
|
| 1275 |
accumulated_content = ""
|
|
@@ -1495,6 +1555,10 @@ def create_demo_prep_interface():
|
|
| 1495 |
workflow_updates += f"💡 Next: Click 'Create DDL' to generate database schema\n"
|
| 1496 |
demo_builder_instance.workflow_updates = workflow_updates
|
| 1497 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1498 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1499 |
|
| 1500 |
elif demo_builder_instance.current_stage == "create":
|
|
@@ -1506,24 +1570,26 @@ def create_demo_prep_interface():
|
|
| 1506 |
)
|
| 1507 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1508 |
|
| 1509 |
-
#
|
| 1510 |
-
|
| 1511 |
-
|
| 1512 |
-
|
| 1513 |
-
|
| 1514 |
-
|
| 1515 |
-
|
| 1516 |
-
|
| 1517 |
-
|
| 1518 |
-
|
| 1519 |
-
|
| 1520 |
-
|
| 1521 |
-
|
| 1522 |
-
|
| 1523 |
-
|
| 1524 |
-
|
| 1525 |
-
|
| 1526 |
-
|
|
|
|
|
|
|
| 1527 |
|
| 1528 |
schema_prompt = f"""Create ONLY complete SQL DDL statements for a {use_case} demo database.
|
| 1529 |
|
|
@@ -1601,6 +1667,11 @@ Generate complete CREATE TABLE statements with proper Snowflake syntax and depen
|
|
| 1601 |
demo_builder_instance.accumulated_content = accumulated_content
|
| 1602 |
demo_builder_instance.workflow_updates = workflow_updates
|
| 1603 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1604 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1605 |
|
| 1606 |
elif demo_builder_instance.current_stage == "populate":
|
|
@@ -1830,13 +1901,18 @@ Schema Validation: Will be checked next...
|
|
| 1830 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1831 |
|
| 1832 |
print(f"🔍 DEBUG: About to advance from populate to deploy stage")
|
| 1833 |
-
|
| 1834 |
demo_builder_instance.advance_stage()
|
| 1835 |
demo_builder_instance.accumulated_content = accumulated_content
|
| 1836 |
demo_builder_instance.workflow_updates = workflow_updates
|
| 1837 |
print(f"🔍 DEBUG: Advanced to stage: {demo_builder_instance.current_stage}")
|
| 1838 |
print(f"🔍 DEBUG: Stage status: {demo_builder_instance.stage_status}")
|
| 1839 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1840 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1841 |
return # Exit populate stage immediately after advancing
|
| 1842 |
|
|
@@ -1876,6 +1952,7 @@ Schema Validation: Will be checked next...
|
|
| 1876 |
company_name,
|
| 1877 |
use_case,
|
| 1878 |
demo_builder_instance.schema_generation_results,
|
|
|
|
| 1879 |
)
|
| 1880 |
)
|
| 1881 |
|
|
@@ -1984,6 +2061,7 @@ Schema Validation: Will be checked next...
|
|
| 1984 |
schema=schema_name,
|
| 1985 |
company_name=company_name,
|
| 1986 |
use_case=use_case,
|
|
|
|
| 1987 |
progress_callback=ts_detailed_callback
|
| 1988 |
)
|
| 1989 |
deployment_complete = True
|
|
@@ -2018,25 +2096,9 @@ Schema Validation: Will be checked next...
|
|
| 2018 |
demo_builder_instance.deployment_status += ts_progress_msg
|
| 2019 |
workflow_updates += ts_progress_msg
|
| 2020 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 2021 |
-
|
| 2022 |
-
# Add detailed progress messages to main window after deployment
|
| 2023 |
-
workflow_updates += "\nBuilding relationships...\n"
|
| 2024 |
-
workflow_updates += "Creating tables...\n"
|
| 2025 |
-
|
| 2026 |
-
# Show detailed table creation info if available
|
| 2027 |
-
if 'tables' in ts_results and ts_results['tables']:
|
| 2028 |
-
workflow_updates += f"Generated joins between tables\n"
|
| 2029 |
-
for table_name in ts_results['tables']:
|
| 2030 |
-
workflow_updates += f" • {table_name} created successfully\n"
|
| 2031 |
-
workflow_updates += f"Tables created successfully ({len(ts_results['tables'])} total)\n"
|
| 2032 |
-
else:
|
| 2033 |
-
workflow_updates += "Generated joins between tables\n"
|
| 2034 |
-
workflow_updates += "Tables created successfully\n"
|
| 2035 |
|
| 2036 |
-
|
| 2037 |
-
|
| 2038 |
-
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 2039 |
-
|
| 2040 |
if ts_results['success']:
|
| 2041 |
ts_msg = "\nThoughtSpot metadata deployment completed!\n\n"
|
| 2042 |
ts_msg += "Created ThoughtSpot objects:\n"
|
|
@@ -2140,9 +2202,19 @@ Schema Validation: Will be checked next...
|
|
| 2140 |
|
| 2141 |
demo_builder_instance.advance_stage()
|
| 2142 |
demo_builder_instance.accumulated_content = accumulated_content
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2143 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 2144 |
|
| 2145 |
except Exception as e:
|
|
|
|
|
|
|
|
|
|
| 2146 |
error_msg = f"❌ Deployment error: {str(e)}\n"
|
| 2147 |
demo_builder_instance.deployment_status += error_msg
|
| 2148 |
workflow_updates += error_msg
|
|
@@ -2158,7 +2230,7 @@ Schema Validation: Will be checked next...
|
|
| 2158 |
yield accumulated_content, workflow_updates, None, "Research", False
|
| 2159 |
|
| 2160 |
# Redo functionality
|
| 2161 |
-
def redo_latest_stage(demo_builder_instance, llm_provider, url, use_case, mode="expert", stop_deploy=False, temperature=0.3, max_tokens=4000):
|
| 2162 |
if (
|
| 2163 |
demo_builder_instance
|
| 2164 |
and demo_builder_instance.current_stage != "research"
|
|
@@ -2173,7 +2245,7 @@ Schema Validation: Will be checked next...
|
|
| 2173 |
|
| 2174 |
# Re-run the workflow with same format as button_click_handler
|
| 2175 |
for result in progressive_workflow_handler(
|
| 2176 |
-
url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens
|
| 2177 |
):
|
| 2178 |
if len(result) == 5:
|
| 2179 |
# New format with workflow_updates
|
|
@@ -2213,7 +2285,7 @@ Schema Validation: Will be checked next...
|
|
| 2213 |
|
| 2214 |
url_input = gr.Textbox(
|
| 2215 |
placeholder="https://www.company.com",
|
| 2216 |
-
value="
|
| 2217 |
label="Company URL",
|
| 2218 |
)
|
| 2219 |
|
|
@@ -2766,7 +2838,7 @@ CREATE TABLE products (
|
|
| 2766 |
)
|
| 2767 |
|
| 2768 |
# TESTING HARNESS - Test population code generation without full workflow
|
| 2769 |
-
def test_population_generation(mock_schema, use_case, llm_provider="
|
| 2770 |
"""Generate population code using mock schema for testing"""
|
| 2771 |
try:
|
| 2772 |
from main_research import MultiLLMResearcher
|
|
@@ -2840,9 +2912,9 @@ SCRIPT REQUIREMENTS:
|
|
| 2840 |
# Legacy CDW deployment function removed - deployment now handled by main workflow
|
| 2841 |
|
| 2842 |
# Main click handler with multi-LLM support - UPDATED to route to AI Output tab
|
| 2843 |
-
def button_click_handler(url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens):
|
| 2844 |
for result in progressive_workflow_handler(
|
| 2845 |
-
url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens
|
| 2846 |
):
|
| 2847 |
if len(result) == 5:
|
| 2848 |
# New format with workflow_updates
|
|
@@ -2873,7 +2945,7 @@ SCRIPT REQUIREMENTS:
|
|
| 2873 |
# Event handlers - UPDATED for new AI Output tab and Results box
|
| 2874 |
research_btn.click(
|
| 2875 |
fn=button_click_handler,
|
| 2876 |
-
inputs=[url_input, industry_dropdown, llm_dropdown, demo_builder, workflow_mode, stop_before_deploy, temperature_slider, max_tokens_input],
|
| 2877 |
outputs=[
|
| 2878 |
ai_output_box, # streaming content goes to AI Output tab
|
| 2879 |
results_box, # dynamic content based on stage/mode
|
|
@@ -2893,10 +2965,10 @@ SCRIPT REQUIREMENTS:
|
|
| 2893 |
)
|
| 2894 |
|
| 2895 |
redo_btn.click(
|
| 2896 |
-
fn=lambda db, llm, url, use_case, mode, stop_deploy, temp, max_tok: [
|
| 2897 |
-
result for result in redo_latest_stage(db, llm, url, use_case, mode, stop_deploy, temp, max_tok)
|
| 2898 |
],
|
| 2899 |
-
inputs=[demo_builder, llm_dropdown, url_input, industry_dropdown, workflow_mode, stop_before_deploy, temperature_slider, max_tokens_input],
|
| 2900 |
outputs=[
|
| 2901 |
ai_output_box, # streaming content goes to AI Output tab
|
| 2902 |
results_box, # dynamic content based on stage/mode
|
|
@@ -3372,7 +3444,7 @@ SCRIPT REQUIREMENTS:
|
|
| 3372 |
|
| 3373 |
def save_settings_handler(
|
| 3374 |
email, ai_model, temperature, max_tokens, company_size, company_url, use_case, data_volume,
|
| 3375 |
-
warehouse, database, ts_url, ts_username, batch_size, thread_count
|
| 3376 |
):
|
| 3377 |
"""Save settings to Supabase database"""
|
| 3378 |
if not email or '@' not in email:
|
|
@@ -3396,7 +3468,8 @@ SCRIPT REQUIREMENTS:
|
|
| 3396 |
"thoughtspot_url": ts_url,
|
| 3397 |
"thoughtspot_username": ts_username,
|
| 3398 |
"batch_size": batch_size,
|
| 3399 |
-
"thread_count": thread_count
|
|
|
|
| 3400 |
}
|
| 3401 |
|
| 3402 |
success = settings_client.save_all_settings(email, settings)
|
|
@@ -3424,6 +3497,7 @@ SCRIPT REQUIREMENTS:
|
|
| 3424 |
"", # ts_username
|
| 3425 |
5000, # batch_size
|
| 3426 |
4, # thread_count
|
|
|
|
| 3427 |
"Settings not loaded - enter valid email", # status
|
| 3428 |
"gpt-4o", # llm_dropdown (main form AI Model)
|
| 3429 |
"" # url_input (main form Company URL)
|
|
@@ -3447,6 +3521,7 @@ SCRIPT REQUIREMENTS:
|
|
| 3447 |
settings.get("thoughtspot_username", ""),
|
| 3448 |
int(settings.get("batch_size", 5000)),
|
| 3449 |
int(settings.get("thread_count", 4)),
|
|
|
|
| 3450 |
f"✅ Settings loaded for {email}",
|
| 3451 |
ai_model, # llm_dropdown (same as default_ai_model)
|
| 3452 |
company_url # url_input (same as default_company_url)
|
|
@@ -3462,6 +3537,7 @@ SCRIPT REQUIREMENTS:
|
|
| 3462 |
"",
|
| 3463 |
5000,
|
| 3464 |
4,
|
|
|
|
| 3465 |
f"❌ Error loading settings: {str(e)}"
|
| 3466 |
]
|
| 3467 |
|
|
@@ -3488,7 +3564,8 @@ SCRIPT REQUIREMENTS:
|
|
| 3488 |
ts_instance_url,
|
| 3489 |
ts_username,
|
| 3490 |
batch_size_slider,
|
| 3491 |
-
thread_count_slider
|
|
|
|
| 3492 |
],
|
| 3493 |
outputs=[settings_status]
|
| 3494 |
)
|
|
@@ -3733,6 +3810,7 @@ SCRIPT REQUIREMENTS:
|
|
| 3733 |
ts_username,
|
| 3734 |
batch_size_slider,
|
| 3735 |
thread_count_slider,
|
|
|
|
| 3736 |
settings_status,
|
| 3737 |
llm_dropdown, # Also update main form AI Model dropdown
|
| 3738 |
url_input # Also update main form Company URL
|
|
|
|
| 371 |
["conn_params.pop('schema', None) # Remove schema to avoid duplicate"]
|
| 372 |
)
|
| 373 |
else:
|
| 374 |
+
# CRITICAL FIX: Add USE SCHEMA command after cursor creation to set schema context
|
| 375 |
+
cleaned_code = cleaned_code.replace(
|
| 376 |
+
"cursor = conn.cursor()",
|
| 377 |
+
f"cursor = conn.cursor()\n cursor.execute('USE SCHEMA \"{schema_name}\"') # Set schema context for INSERTs"
|
| 378 |
+
)
|
| 379 |
|
| 380 |
# Simple and safe schema replacement - just replace the placeholder
|
| 381 |
cleaned_code = cleaned_code.replace("os.getenv('SNOWFLAKE_SCHEMA')", f"'{schema_name}'")
|
| 382 |
cleaned_code = cleaned_code.replace('os.getenv("SNOWFLAKE_SCHEMA")', f'"{schema_name}"')
|
| 383 |
|
| 384 |
+
# Import re module at the top of this section
|
| 385 |
+
import re
|
| 386 |
+
|
| 387 |
# FIX: Remove fake.unique() calls that cause "duplicated values after 1,000 iterations" error
|
| 388 |
cleaned_code = cleaned_code.replace("fake.unique.word()", "fake.word()")
|
| 389 |
cleaned_code = cleaned_code.replace("fake.unique.email()", "fake.email()")
|
| 390 |
cleaned_code = cleaned_code.replace("fake.unique.company()", "fake.company()")
|
| 391 |
|
| 392 |
# FIX: Truncate phone numbers to avoid extension overflow (e.g., '790-923-3730x07350')
|
| 393 |
+
# Handle multiple variations of phone number generation - truncate to 15 chars (max field size)
|
| 394 |
+
cleaned_code = cleaned_code.replace("fake.phone_number()", "fake.phone_number()[:15]")
|
| 395 |
+
cleaned_code = re.sub(r"faker\.phone_number\(\)", "faker.phone_number()[:15]", cleaned_code)
|
| 396 |
+
cleaned_code = re.sub(r"'Phone':\s*fake\.phone_number\(\)", "'Phone': fake.phone_number()[:15]", cleaned_code)
|
| 397 |
+
# Also handle when it's in a larger expression or assignment
|
| 398 |
+
cleaned_code = re.sub(r"(\w+\.phone_number\(\))(?!\[)", r"\1[:15]", cleaned_code)
|
| 399 |
|
| 400 |
# FIX: Convert SQLite-style ? placeholders to Snowflake-style %s placeholders
|
| 401 |
cleaned_code = re.sub(r'\bVALUES\s*\(\?', 'VALUES (%s', cleaned_code)
|
|
|
|
| 574 |
"os": os,
|
| 575 |
"random": random,
|
| 576 |
"datetime": datetime,
|
| 577 |
+
"re": re,
|
| 578 |
"get_snowflake_connection_params": get_snowflake_connection_params,
|
| 579 |
}
|
| 580 |
|
|
|
|
| 586 |
print(f"🔍 DEBUG: Cleaned code preview (first 500 chars):")
|
| 587 |
print(cleaned_code[:500])
|
| 588 |
print("...")
|
| 589 |
+
|
| 590 |
+
# ALWAYS save the full generated code for debugging (not just in DEBUG mode)
|
| 591 |
+
# Save to /tmp for immediate access
|
| 592 |
+
tmp_file = f"/tmp/generated_population_script_{schema_name}.py"
|
| 593 |
+
with open(tmp_file, "w") as f:
|
| 594 |
f.write(cleaned_code)
|
| 595 |
+
print(f"📁 Population script saved to: {tmp_file}")
|
| 596 |
+
|
| 597 |
+
# Also save to logs directory for persistence
|
| 598 |
+
from pathlib import Path
|
| 599 |
+
import datetime
|
| 600 |
+
log_dir = Path("logs") / "population_scripts"
|
| 601 |
+
log_dir.mkdir(parents=True, exist_ok=True)
|
| 602 |
+
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
| 603 |
+
log_file = log_dir / f"population_{schema_name}_{timestamp}.py"
|
| 604 |
+
with open(log_file, "w") as f:
|
| 605 |
+
f.write(cleaned_code)
|
| 606 |
+
print(f"📁 Also saved to: {log_file}")
|
| 607 |
|
| 608 |
# First, validate the syntax before executing
|
| 609 |
try:
|
|
|
|
| 632 |
print("=" * 50)
|
| 633 |
print(f"❌ DATA POPULATION FAILED: {str(e)}")
|
| 634 |
print("=" * 50)
|
| 635 |
+
|
| 636 |
+
# Print detailed traceback
|
| 637 |
+
import traceback
|
| 638 |
+
print("📋 Full traceback:")
|
| 639 |
+
traceback.print_exc()
|
| 640 |
+
|
| 641 |
+
# Show which line of generated code caused the error
|
| 642 |
+
tb = traceback.extract_tb(e.__traceback__)
|
| 643 |
+
for frame in tb:
|
| 644 |
+
if '<population_script>' in frame.filename:
|
| 645 |
+
print(f"\n❌ Error in generated code at line {frame.lineno}:")
|
| 646 |
+
lines = cleaned_code.split('\n')
|
| 647 |
+
start = max(0, frame.lineno - 3)
|
| 648 |
+
end = min(len(lines), frame.lineno + 2)
|
| 649 |
+
for i in range(start, end):
|
| 650 |
+
marker = ">>> " if i == frame.lineno - 1 else " "
|
| 651 |
+
print(f"{marker}{i+1:3d}: {lines[i]}")
|
| 652 |
+
|
| 653 |
return False, f"Population execution failed: {str(e)}"
|
| 654 |
|
| 655 |
def generate_deployment_summary(demo_builder_instance):
|
|
|
|
| 1061 |
info="Primary AI model for all research and generation tasks"
|
| 1062 |
)
|
| 1063 |
default_company_url = gr.Textbox(
|
| 1064 |
+
value="",
|
| 1065 |
label="Default Company URL",
|
| 1066 |
info="Pre-filled company URL for quick starts"
|
| 1067 |
)
|
|
|
|
| 1102 |
label="Schema Prefix",
|
| 1103 |
info="Prefix for schema names (e.g., demo_20250930)"
|
| 1104 |
)
|
| 1105 |
+
naming_prefix_input = gr.Textbox(
|
| 1106 |
+
value="",
|
| 1107 |
+
label="Object Naming Prefix (Optional)",
|
| 1108 |
+
placeholder="e.g., BOONE, SALES, DEMO",
|
| 1109 |
+
info="Custom prefix for all objects. Format: {prefix}_{day+ms}_{objtype}. Leave blank for default naming."
|
| 1110 |
+
)
|
| 1111 |
|
| 1112 |
with gr.TabItem("Admin"):
|
| 1113 |
ts_instance_url = gr.Textbox(
|
|
|
|
| 1311 |
# Progressive workflow with Multi-LLM support
|
| 1312 |
def progressive_workflow_handler(
|
| 1313 |
url, use_case, llm_provider, demo_builder_instance, mode="expert", stop_before_deploy=False,
|
| 1314 |
+
temperature=0.3, max_tokens=4000, naming_prefix=""
|
| 1315 |
):
|
| 1316 |
# Initialize DemoBuilder if none exists
|
| 1317 |
if demo_builder_instance is None:
|
|
|
|
| 1319 |
return "Please enter a company URL", None, "Start Research", False
|
| 1320 |
demo_builder_instance = DemoBuilder(use_case, url.strip())
|
| 1321 |
|
| 1322 |
+
# Initialize demo logger for this session
|
| 1323 |
+
session_logger = get_demo_logger(user_email="demo_user")
|
| 1324 |
+
demo_builder_instance.logger = session_logger
|
| 1325 |
+
session_logger.log_demo_config(
|
| 1326 |
+
use_case=use_case,
|
| 1327 |
+
company=url,
|
| 1328 |
+
company_url=url,
|
| 1329 |
+
industry="TBD",
|
| 1330 |
+
llm_model=llm_provider
|
| 1331 |
+
)
|
| 1332 |
+
|
| 1333 |
# Initialize or continue with existing content
|
| 1334 |
if not hasattr(demo_builder_instance, 'accumulated_content'):
|
| 1335 |
accumulated_content = ""
|
|
|
|
| 1555 |
workflow_updates += f"💡 Next: Click 'Create DDL' to generate database schema\n"
|
| 1556 |
demo_builder_instance.workflow_updates = workflow_updates
|
| 1557 |
|
| 1558 |
+
# Log research stage completion
|
| 1559 |
+
if hasattr(demo_builder_instance, 'logger'):
|
| 1560 |
+
demo_builder_instance.logger.log_stage_completion("research", True)
|
| 1561 |
+
|
| 1562 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1563 |
|
| 1564 |
elif demo_builder_instance.current_stage == "create":
|
|
|
|
| 1570 |
)
|
| 1571 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1572 |
|
| 1573 |
+
# Generate schema name using same convention as ThoughtSpot objects
|
| 1574 |
+
from datetime import datetime as dt
|
| 1575 |
+
now = dt.now()
|
| 1576 |
+
|
| 1577 |
+
# Clean prefix (uppercase, alphanumeric only) - can be empty string
|
| 1578 |
+
if naming_prefix:
|
| 1579 |
+
prefix_clean = re.sub(r'[^a-zA-Z0-9]', '', naming_prefix.upper())
|
| 1580 |
+
else:
|
| 1581 |
+
prefix_clean = ""
|
| 1582 |
+
|
| 1583 |
+
# Short unique timestamp: day (2 digits) + microseconds (6 digits)
|
| 1584 |
+
day = now.strftime('%d')
|
| 1585 |
+
microsec = now.strftime('%f') # 6 digits
|
| 1586 |
+
short_timestamp = f"{day}{microsec}"
|
| 1587 |
+
|
| 1588 |
+
# Base format: PREFIX_DDMICROS (prefix can be empty)
|
| 1589 |
+
if prefix_clean:
|
| 1590 |
+
schema_name = f"{prefix_clean}_{short_timestamp}_scm"
|
| 1591 |
+
else:
|
| 1592 |
+
schema_name = f"{short_timestamp}_scm"
|
| 1593 |
|
| 1594 |
schema_prompt = f"""Create ONLY complete SQL DDL statements for a {use_case} demo database.
|
| 1595 |
|
|
|
|
| 1667 |
demo_builder_instance.accumulated_content = accumulated_content
|
| 1668 |
demo_builder_instance.workflow_updates = workflow_updates
|
| 1669 |
|
| 1670 |
+
# Log DDL stage completion and save DDL
|
| 1671 |
+
if hasattr(demo_builder_instance, 'logger'):
|
| 1672 |
+
demo_builder_instance.logger.log_ddl(schema_results, demo_builder_instance.schema_name if hasattr(demo_builder_instance, 'schema_name') else None)
|
| 1673 |
+
demo_builder_instance.logger.log_stage_completion("ddl", True)
|
| 1674 |
+
|
| 1675 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1676 |
|
| 1677 |
elif demo_builder_instance.current_stage == "populate":
|
|
|
|
| 1901 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1902 |
|
| 1903 |
print(f"🔍 DEBUG: About to advance from populate to deploy stage")
|
| 1904 |
+
|
| 1905 |
demo_builder_instance.advance_stage()
|
| 1906 |
demo_builder_instance.accumulated_content = accumulated_content
|
| 1907 |
demo_builder_instance.workflow_updates = workflow_updates
|
| 1908 |
print(f"🔍 DEBUG: Advanced to stage: {demo_builder_instance.current_stage}")
|
| 1909 |
print(f"🔍 DEBUG: Stage status: {demo_builder_instance.stage_status}")
|
| 1910 |
+
|
| 1911 |
+
# Log population code stage completion
|
| 1912 |
+
if hasattr(demo_builder_instance, 'logger') and hasattr(demo_builder_instance, 'population_code'):
|
| 1913 |
+
demo_builder_instance.logger.log_population_code(demo_builder_instance.population_code)
|
| 1914 |
+
demo_builder_instance.logger.log_stage_completion("population_code", True)
|
| 1915 |
+
|
| 1916 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 1917 |
return # Exit populate stage immediately after advancing
|
| 1918 |
|
|
|
|
| 1952 |
company_name,
|
| 1953 |
use_case,
|
| 1954 |
demo_builder_instance.schema_generation_results,
|
| 1955 |
+
naming_prefix=naming_prefix
|
| 1956 |
)
|
| 1957 |
)
|
| 1958 |
|
|
|
|
| 2061 |
schema=schema_name,
|
| 2062 |
company_name=company_name,
|
| 2063 |
use_case=use_case,
|
| 2064 |
+
naming_prefix=naming_prefix or None,
|
| 2065 |
progress_callback=ts_detailed_callback
|
| 2066 |
)
|
| 2067 |
deployment_complete = True
|
|
|
|
| 2096 |
demo_builder_instance.deployment_status += ts_progress_msg
|
| 2097 |
workflow_updates += ts_progress_msg
|
| 2098 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2099 |
|
| 2100 |
+
# Deployment progress is shown live during execution - no duplicate summary needed
|
| 2101 |
+
|
|
|
|
|
|
|
| 2102 |
if ts_results['success']:
|
| 2103 |
ts_msg = "\nThoughtSpot metadata deployment completed!\n\n"
|
| 2104 |
ts_msg += "Created ThoughtSpot objects:\n"
|
|
|
|
| 2202 |
|
| 2203 |
demo_builder_instance.advance_stage()
|
| 2204 |
demo_builder_instance.accumulated_content = accumulated_content
|
| 2205 |
+
|
| 2206 |
+
# Log deployment stage completion
|
| 2207 |
+
if hasattr(demo_builder_instance, 'logger'):
|
| 2208 |
+
deploy_success = deploy_success if 'deploy_success' in locals() else False
|
| 2209 |
+
demo_builder_instance.logger.log_stage_completion("deploy", deploy_success,
|
| 2210 |
+
details=f"Schema: {schema_name if 'schema_name' in locals() else 'N/A'}")
|
| 2211 |
+
|
| 2212 |
yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
|
| 2213 |
|
| 2214 |
except Exception as e:
|
| 2215 |
+
# Log deployment error
|
| 2216 |
+
if hasattr(demo_builder_instance, 'logger'):
|
| 2217 |
+
demo_builder_instance.logger.log_error(str(e), "deployment", e)
|
| 2218 |
error_msg = f"❌ Deployment error: {str(e)}\n"
|
| 2219 |
demo_builder_instance.deployment_status += error_msg
|
| 2220 |
workflow_updates += error_msg
|
|
|
|
| 2230 |
yield accumulated_content, workflow_updates, None, "Research", False
|
| 2231 |
|
| 2232 |
# Redo functionality
|
| 2233 |
+
def redo_latest_stage(demo_builder_instance, llm_provider, url, use_case, mode="expert", stop_deploy=False, temperature=0.3, max_tokens=4000, naming_prefix=""):
|
| 2234 |
if (
|
| 2235 |
demo_builder_instance
|
| 2236 |
and demo_builder_instance.current_stage != "research"
|
|
|
|
| 2245 |
|
| 2246 |
# Re-run the workflow with same format as button_click_handler
|
| 2247 |
for result in progressive_workflow_handler(
|
| 2248 |
+
url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens, naming_prefix
|
| 2249 |
):
|
| 2250 |
if len(result) == 5:
|
| 2251 |
# New format with workflow_updates
|
|
|
|
| 2285 |
|
| 2286 |
url_input = gr.Textbox(
|
| 2287 |
placeholder="https://www.company.com",
|
| 2288 |
+
value="",
|
| 2289 |
label="Company URL",
|
| 2290 |
)
|
| 2291 |
|
|
|
|
| 2838 |
)
|
| 2839 |
|
| 2840 |
# TESTING HARNESS - Test population code generation without full workflow
|
| 2841 |
+
def test_population_generation(mock_schema, use_case, llm_provider="gpt-4o"):
|
| 2842 |
"""Generate population code using mock schema for testing"""
|
| 2843 |
try:
|
| 2844 |
from main_research import MultiLLMResearcher
|
|
|
|
| 2912 |
# Legacy CDW deployment function removed - deployment now handled by main workflow
|
| 2913 |
|
| 2914 |
# Main click handler with multi-LLM support - UPDATED to route to AI Output tab
|
| 2915 |
+
def button_click_handler(url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens, naming_prefix=""):
|
| 2916 |
for result in progressive_workflow_handler(
|
| 2917 |
+
url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens, naming_prefix
|
| 2918 |
):
|
| 2919 |
if len(result) == 5:
|
| 2920 |
# New format with workflow_updates
|
|
|
|
| 2945 |
# Event handlers - UPDATED for new AI Output tab and Results box
|
| 2946 |
research_btn.click(
|
| 2947 |
fn=button_click_handler,
|
| 2948 |
+
inputs=[url_input, industry_dropdown, llm_dropdown, demo_builder, workflow_mode, stop_before_deploy, temperature_slider, max_tokens_input, naming_prefix_input],
|
| 2949 |
outputs=[
|
| 2950 |
ai_output_box, # streaming content goes to AI Output tab
|
| 2951 |
results_box, # dynamic content based on stage/mode
|
|
|
|
| 2965 |
)
|
| 2966 |
|
| 2967 |
redo_btn.click(
|
| 2968 |
+
fn=lambda db, llm, url, use_case, mode, stop_deploy, temp, max_tok, naming_prefix: [
|
| 2969 |
+
result for result in redo_latest_stage(db, llm, url, use_case, mode, stop_deploy, temp, max_tok, naming_prefix)
|
| 2970 |
],
|
| 2971 |
+
inputs=[demo_builder, llm_dropdown, url_input, industry_dropdown, workflow_mode, stop_before_deploy, temperature_slider, max_tokens_input, naming_prefix_input],
|
| 2972 |
outputs=[
|
| 2973 |
ai_output_box, # streaming content goes to AI Output tab
|
| 2974 |
results_box, # dynamic content based on stage/mode
|
|
|
|
| 3444 |
|
| 3445 |
def save_settings_handler(
|
| 3446 |
email, ai_model, temperature, max_tokens, company_size, company_url, use_case, data_volume,
|
| 3447 |
+
warehouse, database, ts_url, ts_username, batch_size, thread_count, naming_prefix
|
| 3448 |
):
|
| 3449 |
"""Save settings to Supabase database"""
|
| 3450 |
if not email or '@' not in email:
|
|
|
|
| 3468 |
"thoughtspot_url": ts_url,
|
| 3469 |
"thoughtspot_username": ts_username,
|
| 3470 |
"batch_size": batch_size,
|
| 3471 |
+
"thread_count": thread_count,
|
| 3472 |
+
"naming_prefix": naming_prefix or ""
|
| 3473 |
}
|
| 3474 |
|
| 3475 |
success = settings_client.save_all_settings(email, settings)
|
|
|
|
| 3497 |
"", # ts_username
|
| 3498 |
5000, # batch_size
|
| 3499 |
4, # thread_count
|
| 3500 |
+
"", # naming_prefix
|
| 3501 |
"Settings not loaded - enter valid email", # status
|
| 3502 |
"gpt-4o", # llm_dropdown (main form AI Model)
|
| 3503 |
"" # url_input (main form Company URL)
|
|
|
|
| 3521 |
settings.get("thoughtspot_username", ""),
|
| 3522 |
int(settings.get("batch_size", 5000)),
|
| 3523 |
int(settings.get("thread_count", 4)),
|
| 3524 |
+
settings.get("naming_prefix", ""), # naming_prefix
|
| 3525 |
f"✅ Settings loaded for {email}",
|
| 3526 |
ai_model, # llm_dropdown (same as default_ai_model)
|
| 3527 |
company_url # url_input (same as default_company_url)
|
|
|
|
| 3537 |
"",
|
| 3538 |
5000,
|
| 3539 |
4,
|
| 3540 |
+
"", # naming_prefix
|
| 3541 |
f"❌ Error loading settings: {str(e)}"
|
| 3542 |
]
|
| 3543 |
|
|
|
|
| 3564 |
ts_instance_url,
|
| 3565 |
ts_username,
|
| 3566 |
batch_size_slider,
|
| 3567 |
+
thread_count_slider,
|
| 3568 |
+
naming_prefix_input
|
| 3569 |
],
|
| 3570 |
outputs=[settings_status]
|
| 3571 |
)
|
|
|
|
| 3810 |
ts_username,
|
| 3811 |
batch_size_slider,
|
| 3812 |
thread_count_slider,
|
| 3813 |
+
naming_prefix_input,
|
| 3814 |
settings_status,
|
| 3815 |
llm_dropdown, # Also update main form AI Model dropdown
|
| 3816 |
url_input # Also update main form Company URL
|
supabase_client.py
CHANGED
|
@@ -355,6 +355,7 @@ def load_gradio_settings(email: str) -> Dict[str, Any]:
|
|
| 355 |
"default_data_volume": "Medium (10K rows)",
|
| 356 |
"default_warehouse": "COMPUTE_WH",
|
| 357 |
"default_database": "DEMO_DB",
|
|
|
|
| 358 |
|
| 359 |
# Data Generation Settings
|
| 360 |
"fact_table_size": "10000",
|
|
|
|
| 355 |
"default_data_volume": "Medium (10K rows)",
|
| 356 |
"default_warehouse": "COMPUTE_WH",
|
| 357 |
"default_database": "DEMO_DB",
|
| 358 |
+
"naming_prefix": "", # Object naming prefix
|
| 359 |
|
| 360 |
# Data Generation Settings
|
| 361 |
"fact_table_size": "10000",
|
tests/debug_population_error.py
DELETED
|
@@ -1,245 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
Debug script to isolate the 'name e is not defined' error in population code generation.
|
| 4 |
-
This will help us understand exactly what's happening without running the full app.
|
| 5 |
-
"""
|
| 6 |
-
|
| 7 |
-
import os
|
| 8 |
-
import sys
|
| 9 |
-
from dotenv import load_dotenv
|
| 10 |
-
|
| 11 |
-
# Add the current directory to Python path
|
| 12 |
-
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
| 13 |
-
|
| 14 |
-
# Import the functions we need to test
|
| 15 |
-
try:
|
| 16 |
-
from demo_prep import execute_population_script, extract_python_code, validate_python_syntax
|
| 17 |
-
from schema_utils import generate_schema_constrained_prompt, parse_ddl_schema
|
| 18 |
-
print("✅ Successfully imported required functions")
|
| 19 |
-
except ImportError as e:
|
| 20 |
-
print(f"❌ Import error: {e}")
|
| 21 |
-
sys.exit(1)
|
| 22 |
-
|
| 23 |
-
def test_population_error():
|
| 24 |
-
"""Test the exact scenario that's causing the 'name e is not defined' error"""
|
| 25 |
-
|
| 26 |
-
print("🔍 DEBUG TEST: Isolating population script error")
|
| 27 |
-
print("=" * 60)
|
| 28 |
-
|
| 29 |
-
# Load environment
|
| 30 |
-
load_dotenv()
|
| 31 |
-
|
| 32 |
-
# Sample DDL that should work (based on previous successful runs)
|
| 33 |
-
sample_ddl = """
|
| 34 |
-
CREATE TABLE CHANNELS (
|
| 35 |
-
CHANNELID INT IDENTITY(1,1) PRIMARY KEY,
|
| 36 |
-
NAME VARCHAR(100) NOT NULL,
|
| 37 |
-
TYPE VARCHAR(50) NOT NULL,
|
| 38 |
-
PERFORMANCEMETRICS VARCHAR(500)
|
| 39 |
-
);
|
| 40 |
-
|
| 41 |
-
CREATE TABLE CUSTOMERS (
|
| 42 |
-
CUSTOMERID INT IDENTITY(1,1) PRIMARY KEY,
|
| 43 |
-
NAME VARCHAR(100) NOT NULL,
|
| 44 |
-
EMAIL VARCHAR(100) NOT NULL,
|
| 45 |
-
PHONE VARCHAR(20),
|
| 46 |
-
ADDRESS VARCHAR(200),
|
| 47 |
-
PREFERENCES VARCHAR(500),
|
| 48 |
-
INTERACTIONHISTORY VARCHAR(1000)
|
| 49 |
-
);
|
| 50 |
-
|
| 51 |
-
CREATE TABLE PRODUCTS (
|
| 52 |
-
PRODUCTID INT IDENTITY(1,1) PRIMARY KEY,
|
| 53 |
-
NAME VARCHAR(100) NOT NULL,
|
| 54 |
-
CATEGORY VARCHAR(50) NOT NULL,
|
| 55 |
-
PRICE DECIMAL(10,2) NOT NULL,
|
| 56 |
-
STOCKLEVEL INT NOT NULL
|
| 57 |
-
);
|
| 58 |
-
|
| 59 |
-
CREATE TABLE SALESTRANSACTIONS (
|
| 60 |
-
TRANSACTIONID INT IDENTITY(1,1) PRIMARY KEY,
|
| 61 |
-
CUSTOMERID INT NOT NULL,
|
| 62 |
-
PRODUCTID INT NOT NULL,
|
| 63 |
-
DATE DATE NOT NULL,
|
| 64 |
-
AMOUNT DECIMAL(10,2) NOT NULL,
|
| 65 |
-
CHANNELID INT NOT NULL,
|
| 66 |
-
FOREIGN KEY (CUSTOMERID) REFERENCES CUSTOMERS(CUSTOMERID),
|
| 67 |
-
FOREIGN KEY (PRODUCTID) REFERENCES PRODUCTS(PRODUCTID),
|
| 68 |
-
FOREIGN KEY (CHANNELID) REFERENCES CHANNELS(CHANNELID)
|
| 69 |
-
);
|
| 70 |
-
|
| 71 |
-
CREATE TABLE CUSTOMERINTERACTIONS (
|
| 72 |
-
INTERACTIONID INT IDENTITY(1,1) PRIMARY KEY,
|
| 73 |
-
CUSTOMERID INT NOT NULL,
|
| 74 |
-
CHANNELID INT NOT NULL,
|
| 75 |
-
DATE DATE NOT NULL,
|
| 76 |
-
INTERACTIONTYPE VARCHAR(50) NOT NULL,
|
| 77 |
-
OUTCOME VARCHAR(50) NOT NULL,
|
| 78 |
-
FOREIGN KEY (CUSTOMERID) REFERENCES CUSTOMERS(CUSTOMERID),
|
| 79 |
-
FOREIGN KEY (CHANNELID) REFERENCES CHANNELS(CHANNELID)
|
| 80 |
-
);
|
| 81 |
-
"""
|
| 82 |
-
|
| 83 |
-
print("📋 Step 1: Testing DDL parsing...")
|
| 84 |
-
try:
|
| 85 |
-
schema_info = parse_ddl_schema(sample_ddl)
|
| 86 |
-
print(f"✅ DDL parsed successfully: {len(schema_info)} tables found")
|
| 87 |
-
for table_name, table_info in schema_info.items():
|
| 88 |
-
print(f" - {table_name}: {len(table_info['columns'])} columns")
|
| 89 |
-
except Exception as e:
|
| 90 |
-
print(f"❌ DDL parsing failed: {e}")
|
| 91 |
-
return False
|
| 92 |
-
|
| 93 |
-
print("\n📝 Step 2: Testing prompt generation...")
|
| 94 |
-
try:
|
| 95 |
-
business_context = """
|
| 96 |
-
BUSINESS CONTEXT:
|
| 97 |
-
- Use Case: Sales AI Analyst
|
| 98 |
-
- Target Persona: Sales Operations Manager
|
| 99 |
-
- Business Problem: Identifying sales performance gaps and customer churn risks
|
| 100 |
-
- Demo Objectives: Demonstrate AI-driven sales insights and predictive analytics
|
| 101 |
-
- Success Outcomes: Improved sales forecasting and customer retention
|
| 102 |
-
|
| 103 |
-
MANDATORY CONNECTION CODE:
|
| 104 |
-
from dotenv import load_dotenv
|
| 105 |
-
import os
|
| 106 |
-
import snowflake.connector
|
| 107 |
-
|
| 108 |
-
load_dotenv()
|
| 109 |
-
|
| 110 |
-
from snowflake_auth import get_snowflake_connection_params
|
| 111 |
-
conn_params = get_snowflake_connection_params()
|
| 112 |
-
conn = snowflake.connector.connect(
|
| 113 |
-
**conn_params,
|
| 114 |
-
schema=os.getenv('SNOWFLAKE_SCHEMA')
|
| 115 |
-
)
|
| 116 |
-
|
| 117 |
-
SCRIPT REQUIREMENTS:
|
| 118 |
-
1. Connect to Snowflake using connection code above
|
| 119 |
-
2. Populate tables with realistic data volumes (1000+ rows per table)
|
| 120 |
-
3. Create baseline normal data patterns
|
| 121 |
-
4. NO explanatory text, just executable Python code
|
| 122 |
-
5. Use proper error handling with try-catch blocks
|
| 123 |
-
6. Include data validation to ensure referential integrity
|
| 124 |
-
7. Add progress logging for each table population
|
| 125 |
-
8. Use realistic data patterns that match the business context
|
| 126 |
-
9. Ensure all foreign key relationships are maintained
|
| 127 |
-
"""
|
| 128 |
-
|
| 129 |
-
data_prompt = generate_schema_constrained_prompt(
|
| 130 |
-
schema_info,
|
| 131 |
-
"Sales AI Analyst",
|
| 132 |
-
business_context
|
| 133 |
-
)
|
| 134 |
-
print(f"✅ Prompt generated successfully: {len(data_prompt)} characters")
|
| 135 |
-
except Exception as e:
|
| 136 |
-
print(f"❌ Prompt generation failed: {e}")
|
| 137 |
-
return False
|
| 138 |
-
|
| 139 |
-
print("\n🤖 Step 3: Using sample broken LLM output...")
|
| 140 |
-
# Use a sample broken LLM output that likely causes the "name 'e' is not defined" error
|
| 141 |
-
llm_output = '''```python
|
| 142 |
-
from dotenv import load_dotenv
|
| 143 |
-
import os
|
| 144 |
-
import snowflake.connector
|
| 145 |
-
from snowflake_auth import get_snowflake_connection_params
|
| 146 |
-
from faker import Faker
|
| 147 |
-
import random
|
| 148 |
-
|
| 149 |
-
def main():
|
| 150 |
-
try:
|
| 151 |
-
load_dotenv()
|
| 152 |
-
conn_params = get_snowflake_connection_params()
|
| 153 |
-
conn = snowflake.connector.connect(
|
| 154 |
-
**conn_params,
|
| 155 |
-
schema=os.getenv('SNOWFLAKE_SCHEMA')
|
| 156 |
-
)
|
| 157 |
-
cursor = conn.cursor()
|
| 158 |
-
|
| 159 |
-
# Populate Channels table
|
| 160 |
-
print("Populating Channels...")
|
| 161 |
-
for i in range(100):
|
| 162 |
-
cursor.execute("INSERT INTO CHANNELS (NAME, TYPE) VALUES (?, ?)",
|
| 163 |
-
(f"Channel {i}", "Online"))
|
| 164 |
-
|
| 165 |
-
conn.commit()
|
| 166 |
-
print("Population completed!")
|
| 167 |
-
|
| 168 |
-
# This is the broken part - incomplete try-except block
|
| 169 |
-
except Exception as e:
|
| 170 |
-
print(f"Error: {e}")
|
| 171 |
-
# Missing finally or another except block that references 'e'
|
| 172 |
-
|
| 173 |
-
if __name__ == "__main__":
|
| 174 |
-
main()
|
| 175 |
-
```'''
|
| 176 |
-
|
| 177 |
-
print(f"✅ Using sample LLM output: {len(llm_output)} characters")
|
| 178 |
-
|
| 179 |
-
# Save the output for inspection
|
| 180 |
-
with open("debug_llm_output.txt", "w") as f:
|
| 181 |
-
f.write(llm_output)
|
| 182 |
-
print("📁 Saved sample LLM output to debug_llm_output.txt")
|
| 183 |
-
|
| 184 |
-
print("\n🔍 Step 4: Testing syntax validation...")
|
| 185 |
-
try:
|
| 186 |
-
syntax_valid, syntax_message = validate_python_syntax(llm_output)
|
| 187 |
-
print(f"Syntax validation result: {syntax_valid}")
|
| 188 |
-
print(f"Syntax message: {syntax_message}")
|
| 189 |
-
except Exception as e:
|
| 190 |
-
print(f"❌ Syntax validation failed: {e}")
|
| 191 |
-
print(f"❌ This might be our 'name e is not defined' error!")
|
| 192 |
-
return False
|
| 193 |
-
|
| 194 |
-
print("\n🔧 Step 5: Testing code extraction...")
|
| 195 |
-
try:
|
| 196 |
-
extracted_code = extract_python_code(llm_output)
|
| 197 |
-
print(f"✅ Code extracted: {len(extracted_code)} characters")
|
| 198 |
-
|
| 199 |
-
# Save the extracted code for inspection
|
| 200 |
-
with open("debug_extracted_code.py", "w") as f:
|
| 201 |
-
f.write(extracted_code)
|
| 202 |
-
print("📁 Saved extracted code to debug_extracted_code.py")
|
| 203 |
-
|
| 204 |
-
except Exception as e:
|
| 205 |
-
print(f"❌ Code extraction failed: {e}")
|
| 206 |
-
return False
|
| 207 |
-
|
| 208 |
-
print("\n🚀 Step 6: Testing population script execution...")
|
| 209 |
-
try:
|
| 210 |
-
# Use a test schema name
|
| 211 |
-
test_schema = "TEST_DEBUG_SCHEMA"
|
| 212 |
-
|
| 213 |
-
print(f" Testing with schema: {test_schema}")
|
| 214 |
-
success, message = execute_population_script(llm_output, test_schema)
|
| 215 |
-
|
| 216 |
-
if success:
|
| 217 |
-
print(f"✅ Population script executed successfully!")
|
| 218 |
-
else:
|
| 219 |
-
print(f"❌ Population script failed: {message}")
|
| 220 |
-
print("❌ This is likely where our error occurs!")
|
| 221 |
-
|
| 222 |
-
except Exception as e:
|
| 223 |
-
print(f"❌ Population script execution failed: {e}")
|
| 224 |
-
print(f"❌ Error type: {type(e).__name__}")
|
| 225 |
-
import traceback
|
| 226 |
-
print("❌ Full traceback:")
|
| 227 |
-
traceback.print_exc()
|
| 228 |
-
return False
|
| 229 |
-
|
| 230 |
-
print("\n" + "=" * 60)
|
| 231 |
-
print("🎯 DEBUG TEST COMPLETE")
|
| 232 |
-
print("Check the generated files:")
|
| 233 |
-
print(" - debug_llm_output.txt (raw LLM response)")
|
| 234 |
-
print(" - debug_extracted_code.py (extracted Python code)")
|
| 235 |
-
print("=" * 60)
|
| 236 |
-
|
| 237 |
-
return True
|
| 238 |
-
|
| 239 |
-
if __name__ == "__main__":
|
| 240 |
-
print("🧪 Starting isolated debug test for population error...")
|
| 241 |
-
success = test_population_error()
|
| 242 |
-
if success:
|
| 243 |
-
print("✅ Debug test completed successfully!")
|
| 244 |
-
else:
|
| 245 |
-
print("❌ Debug test failed - this should help identify the issue!")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/debug_schema_utils.py
DELETED
|
@@ -1,106 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
Debug script to isolate the 'name e is not defined' error in schema_utils.py
|
| 4 |
-
"""
|
| 5 |
-
|
| 6 |
-
import os
|
| 7 |
-
import sys
|
| 8 |
-
from dotenv import load_dotenv
|
| 9 |
-
|
| 10 |
-
# Add the current directory to Python path
|
| 11 |
-
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
| 12 |
-
|
| 13 |
-
# Import the functions we need to test
|
| 14 |
-
try:
|
| 15 |
-
from schema_utils import generate_schema_constrained_prompt, parse_ddl_schema
|
| 16 |
-
print("✅ Successfully imported schema_utils functions")
|
| 17 |
-
except ImportError as e:
|
| 18 |
-
print(f"❌ Import error: {e}")
|
| 19 |
-
sys.exit(1)
|
| 20 |
-
|
| 21 |
-
def test_schema_utils_error():
|
| 22 |
-
"""Test the generate_schema_constrained_prompt function specifically"""
|
| 23 |
-
|
| 24 |
-
print("🔍 DEBUG TEST: Isolating schema_utils error")
|
| 25 |
-
print("=" * 60)
|
| 26 |
-
|
| 27 |
-
# Load environment
|
| 28 |
-
load_dotenv()
|
| 29 |
-
|
| 30 |
-
# Sample DDL
|
| 31 |
-
sample_ddl = """
|
| 32 |
-
CREATE TABLE CHANNELS (
|
| 33 |
-
CHANNELID INT IDENTITY(1,1) PRIMARY KEY,
|
| 34 |
-
NAME VARCHAR(100) NOT NULL,
|
| 35 |
-
TYPE VARCHAR(50) NOT NULL,
|
| 36 |
-
PERFORMANCEMETRICS VARCHAR(500)
|
| 37 |
-
);
|
| 38 |
-
|
| 39 |
-
CREATE TABLE CUSTOMERS (
|
| 40 |
-
CUSTOMERID INT IDENTITY(1,1) PRIMARY KEY,
|
| 41 |
-
NAME VARCHAR(100) NOT NULL,
|
| 42 |
-
EMAIL VARCHAR(100) NOT NULL,
|
| 43 |
-
PHONE VARCHAR(20),
|
| 44 |
-
ADDRESS VARCHAR(200),
|
| 45 |
-
PREFERENCES VARCHAR(500),
|
| 46 |
-
INTERACTIONHISTORY VARCHAR(1000)
|
| 47 |
-
);
|
| 48 |
-
"""
|
| 49 |
-
|
| 50 |
-
print("📋 Step 1: Testing DDL parsing...")
|
| 51 |
-
try:
|
| 52 |
-
schema_info = parse_ddl_schema(sample_ddl)
|
| 53 |
-
print(f"✅ DDL parsed successfully: {len(schema_info)} tables found")
|
| 54 |
-
for table_name, table_info in schema_info.items():
|
| 55 |
-
print(f" - {table_name}: {len(table_info['columns'])} columns")
|
| 56 |
-
print(f" Columns: {[col['name'] for col in table_info['columns']]}")
|
| 57 |
-
except Exception as e:
|
| 58 |
-
print(f"❌ DDL parsing failed: {e}")
|
| 59 |
-
import traceback
|
| 60 |
-
print("❌ Full traceback:")
|
| 61 |
-
traceback.print_exc()
|
| 62 |
-
return False
|
| 63 |
-
|
| 64 |
-
print("\n📝 Step 2: Testing prompt generation...")
|
| 65 |
-
try:
|
| 66 |
-
business_context = """
|
| 67 |
-
BUSINESS CONTEXT:
|
| 68 |
-
- Use Case: Sales AI Analyst
|
| 69 |
-
- Target Persona: Sales Operations Manager
|
| 70 |
-
- Business Problem: Identifying sales performance gaps
|
| 71 |
-
"""
|
| 72 |
-
|
| 73 |
-
print(" Calling generate_schema_constrained_prompt...")
|
| 74 |
-
data_prompt = generate_schema_constrained_prompt(
|
| 75 |
-
schema_info,
|
| 76 |
-
"Sales AI Analyst",
|
| 77 |
-
business_context
|
| 78 |
-
)
|
| 79 |
-
print(f"✅ Prompt generated successfully: {len(data_prompt)} characters")
|
| 80 |
-
|
| 81 |
-
# Save the prompt for inspection
|
| 82 |
-
with open("debug_generated_prompt.txt", "w") as f:
|
| 83 |
-
f.write(data_prompt)
|
| 84 |
-
print("📁 Saved generated prompt to debug_generated_prompt.txt")
|
| 85 |
-
|
| 86 |
-
except Exception as e:
|
| 87 |
-
print(f"❌ Prompt generation failed: {e}")
|
| 88 |
-
print(f"❌ Error type: {type(e).__name__}")
|
| 89 |
-
import traceback
|
| 90 |
-
print("❌ Full traceback:")
|
| 91 |
-
traceback.print_exc()
|
| 92 |
-
return False
|
| 93 |
-
|
| 94 |
-
print("\n" + "=" * 60)
|
| 95 |
-
print("🎯 SCHEMA_UTILS DEBUG TEST COMPLETE")
|
| 96 |
-
print("=" * 60)
|
| 97 |
-
|
| 98 |
-
return True
|
| 99 |
-
|
| 100 |
-
if __name__ == "__main__":
|
| 101 |
-
print("🧪 Starting isolated debug test for schema_utils error...")
|
| 102 |
-
success = test_schema_utils_error()
|
| 103 |
-
if success:
|
| 104 |
-
print("✅ Debug test completed successfully!")
|
| 105 |
-
else:
|
| 106 |
-
print("❌ Debug test failed - this should help identify the issue!")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_connection_success.py
DELETED
|
@@ -1,123 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
SUCCESS-ONLY ThoughtSpot Connection Test
|
| 4 |
-
GUARANTEED SUCCESS - Connection creation only
|
| 5 |
-
"""
|
| 6 |
-
|
| 7 |
-
import os
|
| 8 |
-
import sys
|
| 9 |
-
# Add parent directory to path
|
| 10 |
-
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
| 11 |
-
|
| 12 |
-
from dotenv import load_dotenv
|
| 13 |
-
from thoughtspot_deployer import ThoughtSpotDeployer
|
| 14 |
-
|
| 15 |
-
load_dotenv()
|
| 16 |
-
|
| 17 |
-
def test_connection_success():
|
| 18 |
-
"""Test ONLY connection creation - GUARANTEED SUCCESS"""
|
| 19 |
-
|
| 20 |
-
print("🎯 THOUGHTSPOT CONNECTION SUCCESS TEST")
|
| 21 |
-
print("=" * 55)
|
| 22 |
-
print("🎉 This test focuses ONLY on what works perfectly!")
|
| 23 |
-
|
| 24 |
-
schema_name = "20250918_001404_THOUG_SAL"
|
| 25 |
-
database = "DEMOBUILD"
|
| 26 |
-
|
| 27 |
-
print(f"📊 Testing with schema: {schema_name}")
|
| 28 |
-
print(f"📊 Database: {database}")
|
| 29 |
-
print()
|
| 30 |
-
|
| 31 |
-
try:
|
| 32 |
-
# Initialize deployer
|
| 33 |
-
deployer = ThoughtSpotDeployer()
|
| 34 |
-
print("✅ ThoughtSpot deployer initialized")
|
| 35 |
-
|
| 36 |
-
# Test authentication
|
| 37 |
-
print("\n🔐 Testing ThoughtSpot authentication...")
|
| 38 |
-
if deployer.authenticate():
|
| 39 |
-
print("✅ ThoughtSpot authentication SUCCESSFUL")
|
| 40 |
-
else:
|
| 41 |
-
print("❌ ThoughtSpot authentication failed")
|
| 42 |
-
return False
|
| 43 |
-
|
| 44 |
-
# Test connection creation ONLY
|
| 45 |
-
print("\n🔗 Testing connection creation...")
|
| 46 |
-
|
| 47 |
-
# Create connection using the internal method directly
|
| 48 |
-
from datetime import datetime
|
| 49 |
-
connection_name = f"success_conn_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
|
| 50 |
-
|
| 51 |
-
print(f"🔗 Creating connection: {connection_name}")
|
| 52 |
-
print(f" Account: '{deployer.sf_account}'")
|
| 53 |
-
print(f" User: '{deployer.sf_user}'")
|
| 54 |
-
print(f" Database: '{database}'")
|
| 55 |
-
|
| 56 |
-
# Create the connection TML
|
| 57 |
-
connection_tml_yaml = deployer.create_connection_tml(connection_name, database)
|
| 58 |
-
|
| 59 |
-
# Deploy the connection
|
| 60 |
-
response = deployer.session.post(
|
| 61 |
-
f"{deployer.base_url}/api/rest/2.0/metadata/tml/import",
|
| 62 |
-
json={
|
| 63 |
-
"metadata_tmls": [connection_tml_yaml],
|
| 64 |
-
"import_policy": "PARTIAL"
|
| 65 |
-
}
|
| 66 |
-
)
|
| 67 |
-
|
| 68 |
-
print(f" Response status: {response.status_code}")
|
| 69 |
-
|
| 70 |
-
if response.status_code == 200:
|
| 71 |
-
result = response.json()
|
| 72 |
-
if isinstance(result, list) and len(result) > 0:
|
| 73 |
-
if result[0].get('response', {}).get('status', {}).get('status_code') == 'OK':
|
| 74 |
-
connection_guid = result[0].get('response', {}).get('header', {}).get('id_guid')
|
| 75 |
-
|
| 76 |
-
print("🎉" * 20)
|
| 77 |
-
print("🎯 CONNECTION CREATION SUCCESS!")
|
| 78 |
-
print(f"✅ Connection Name: {connection_name}")
|
| 79 |
-
print(f"✅ Connection GUID: {connection_guid}")
|
| 80 |
-
print(f"✅ Schema Access: {schema_name}")
|
| 81 |
-
print(f"✅ Authentication: KEY_PAIR working")
|
| 82 |
-
print(f"✅ API Integration: PERFECT")
|
| 83 |
-
print("🎉" * 20)
|
| 84 |
-
|
| 85 |
-
return True
|
| 86 |
-
else:
|
| 87 |
-
error_msg = result[0].get('response', {}).get('status', {}).get('error_message', 'Unknown error')
|
| 88 |
-
print(f"❌ Connection creation failed: {error_msg}")
|
| 89 |
-
return False
|
| 90 |
-
else:
|
| 91 |
-
print(f"❌ Unexpected response format: {result}")
|
| 92 |
-
return False
|
| 93 |
-
else:
|
| 94 |
-
try:
|
| 95 |
-
error_response = response.json()
|
| 96 |
-
print(f"❌ HTTP {response.status_code}: {error_response}")
|
| 97 |
-
except:
|
| 98 |
-
print(f"❌ HTTP {response.status_code}: {response.text}")
|
| 99 |
-
return False
|
| 100 |
-
|
| 101 |
-
except Exception as e:
|
| 102 |
-
print(f"❌ Test failed with exception: {str(e)}")
|
| 103 |
-
import traceback
|
| 104 |
-
traceback.print_exc()
|
| 105 |
-
return False
|
| 106 |
-
|
| 107 |
-
def main():
|
| 108 |
-
"""Run the success test"""
|
| 109 |
-
success = test_connection_success()
|
| 110 |
-
|
| 111 |
-
print("\n" + "="*55)
|
| 112 |
-
if success:
|
| 113 |
-
print("🏆 OVERALL RESULT: COMPLETE SUCCESS! 🏆")
|
| 114 |
-
print("🚀 ThoughtSpot integration is working perfectly!")
|
| 115 |
-
print("🎯 Ready for production deployment!")
|
| 116 |
-
else:
|
| 117 |
-
print("❌ OVERALL RESULT: Something failed")
|
| 118 |
-
print("="*55)
|
| 119 |
-
|
| 120 |
-
return success
|
| 121 |
-
|
| 122 |
-
if __name__ == "__main__":
|
| 123 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_core_success.py
DELETED
|
@@ -1,150 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
CORE SUCCESS ThoughtSpot Test
|
| 4 |
-
Skip problematic TIMESTAMP columns, focus on core columns
|
| 5 |
-
Connection + Tables + Model = SUCCESS
|
| 6 |
-
"""
|
| 7 |
-
|
| 8 |
-
import os
|
| 9 |
-
import sys
|
| 10 |
-
# Add parent directory to path
|
| 11 |
-
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
| 12 |
-
|
| 13 |
-
from dotenv import load_dotenv
|
| 14 |
-
from thoughtspot_deployer import ThoughtSpotDeployer
|
| 15 |
-
|
| 16 |
-
load_dotenv()
|
| 17 |
-
|
| 18 |
-
def test_core_success():
|
| 19 |
-
"""Test with core columns only (skip CREATEDAT timestamps)"""
|
| 20 |
-
|
| 21 |
-
print("🎯 THOUGHTSPOT CORE SUCCESS TEST")
|
| 22 |
-
print("=" * 55)
|
| 23 |
-
print("🎉 Using CORE columns only (skip CREATEDAT)!")
|
| 24 |
-
|
| 25 |
-
schema_name = "20250918_001404_THOUG_SAL"
|
| 26 |
-
database = "DEMOBUILD"
|
| 27 |
-
|
| 28 |
-
# Core DDL - skip CREATEDAT columns that cause timestamp issues
|
| 29 |
-
core_ddl = """
|
| 30 |
-
CREATE TABLE CUSTOMERS (
|
| 31 |
-
CUSTOMERID NUMBER(38,0) NOT NULL,
|
| 32 |
-
FIRSTNAME VARCHAR(50) NOT NULL,
|
| 33 |
-
LASTNAME VARCHAR(50) NOT NULL,
|
| 34 |
-
EMAIL VARCHAR(100) NOT NULL,
|
| 35 |
-
PHONE VARCHAR(20) NOT NULL,
|
| 36 |
-
ADDRESS VARCHAR(255) NOT NULL,
|
| 37 |
-
CITY VARCHAR(50) NOT NULL,
|
| 38 |
-
STATE VARCHAR(50) NOT NULL,
|
| 39 |
-
ZIPCODE VARCHAR(10) NOT NULL,
|
| 40 |
-
COUNTRY VARCHAR(50) NOT NULL
|
| 41 |
-
);
|
| 42 |
-
|
| 43 |
-
CREATE TABLE PRODUCTS (
|
| 44 |
-
PRODUCTID NUMBER(38,0) NOT NULL,
|
| 45 |
-
PRODUCTNAME VARCHAR(100) NOT NULL,
|
| 46 |
-
DESCRIPTION VARCHAR(255) NOT NULL,
|
| 47 |
-
PRICE NUMBER(10,2) NOT NULL,
|
| 48 |
-
STOCKQUANTITY NUMBER(38,0) NOT NULL
|
| 49 |
-
);
|
| 50 |
-
|
| 51 |
-
CREATE TABLE ORDERS (
|
| 52 |
-
ORDERID NUMBER(38,0) NOT NULL,
|
| 53 |
-
CUSTOMERID NUMBER(38,0) NOT NULL,
|
| 54 |
-
ORDERDATE DATE NOT NULL,
|
| 55 |
-
TOTALAMOUNT NUMBER(10,2) NOT NULL,
|
| 56 |
-
STATUS VARCHAR(50) NOT NULL
|
| 57 |
-
);
|
| 58 |
-
"""
|
| 59 |
-
|
| 60 |
-
print(f"📊 Testing with schema: {schema_name}")
|
| 61 |
-
print(f"📊 Database: {database}")
|
| 62 |
-
print(f"📊 Tables: 3 core tables (no CREATEDAT columns)")
|
| 63 |
-
print()
|
| 64 |
-
|
| 65 |
-
try:
|
| 66 |
-
# Initialize deployer
|
| 67 |
-
deployer = ThoughtSpotDeployer()
|
| 68 |
-
print("✅ ThoughtSpot deployer initialized")
|
| 69 |
-
|
| 70 |
-
# Test authentication
|
| 71 |
-
print("\n🔐 Testing ThoughtSpot authentication...")
|
| 72 |
-
if deployer.authenticate():
|
| 73 |
-
print("✅ ThoughtSpot authentication SUCCESSFUL")
|
| 74 |
-
else:
|
| 75 |
-
print("❌ ThoughtSpot authentication failed")
|
| 76 |
-
return False
|
| 77 |
-
|
| 78 |
-
# Test FULL deployment
|
| 79 |
-
print("\n🚀 Testing CORE deployment (Connection + Tables + Model)...")
|
| 80 |
-
|
| 81 |
-
results = deployer.deploy_all(
|
| 82 |
-
ddl=core_ddl,
|
| 83 |
-
database=database,
|
| 84 |
-
schema=schema_name
|
| 85 |
-
)
|
| 86 |
-
|
| 87 |
-
print(f"\n📋 RESULTS:")
|
| 88 |
-
|
| 89 |
-
# Connection
|
| 90 |
-
if results.get('connection'):
|
| 91 |
-
print(f"✅ Connection: {results['connection']}")
|
| 92 |
-
else:
|
| 93 |
-
print("❌ Connection: Failed")
|
| 94 |
-
|
| 95 |
-
# Tables
|
| 96 |
-
if results.get('tables'):
|
| 97 |
-
print(f"✅ Tables: {len(results['tables'])} created")
|
| 98 |
-
for table in results['tables']:
|
| 99 |
-
print(f" 📊 {table}")
|
| 100 |
-
else:
|
| 101 |
-
print("❌ Tables: None created")
|
| 102 |
-
if results.get('errors'):
|
| 103 |
-
print(" 🔍 First few errors:")
|
| 104 |
-
for error in results['errors'][:2]:
|
| 105 |
-
print(f" • {error[:80]}...")
|
| 106 |
-
|
| 107 |
-
# Model
|
| 108 |
-
if results.get('model'):
|
| 109 |
-
print(f"✅ Model: {results['model']}")
|
| 110 |
-
else:
|
| 111 |
-
print("❌ Model: None created")
|
| 112 |
-
|
| 113 |
-
# Success evaluation
|
| 114 |
-
connection_success = bool(results.get('connection'))
|
| 115 |
-
tables_success = len(results.get('tables', [])) > 0
|
| 116 |
-
model_success = bool(results.get('model'))
|
| 117 |
-
|
| 118 |
-
print(f"\n🎯 SUCCESS BREAKDOWN:")
|
| 119 |
-
print(f" Connection: {'✅' if connection_success else '❌'}")
|
| 120 |
-
print(f" Tables: {'✅' if tables_success else '❌'}")
|
| 121 |
-
print(f" Model: {'✅' if model_success else '❌'}")
|
| 122 |
-
|
| 123 |
-
overall_success = connection_success and tables_success
|
| 124 |
-
|
| 125 |
-
print("\n" + "="*55)
|
| 126 |
-
if overall_success:
|
| 127 |
-
print("🏆 OVERALL RESULT: SUCCESS! 🏆")
|
| 128 |
-
print("🎉 Connection + Tables working!")
|
| 129 |
-
if model_success:
|
| 130 |
-
print("🎯 COMPLETE SUCCESS - Model too!")
|
| 131 |
-
else:
|
| 132 |
-
print("🔧 Model needs relationships, but CORE SUCCESS!")
|
| 133 |
-
else:
|
| 134 |
-
print("❌ OVERALL RESULT: Still working on it...")
|
| 135 |
-
print("="*55)
|
| 136 |
-
|
| 137 |
-
return overall_success
|
| 138 |
-
|
| 139 |
-
except Exception as e:
|
| 140 |
-
print(f"❌ Test failed with exception: {str(e)}")
|
| 141 |
-
import traceback
|
| 142 |
-
traceback.print_exc()
|
| 143 |
-
return False
|
| 144 |
-
|
| 145 |
-
def main():
|
| 146 |
-
"""Run the core success test"""
|
| 147 |
-
return test_core_success()
|
| 148 |
-
|
| 149 |
-
if __name__ == "__main__":
|
| 150 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_deployment_flow.py
DELETED
|
@@ -1,54 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
Test the updated deployment flow with schema permissions
|
| 4 |
-
"""
|
| 5 |
-
|
| 6 |
-
import os
|
| 7 |
-
from dotenv import load_dotenv
|
| 8 |
-
|
| 9 |
-
# Load environment variables (you may need to copy .env from sandbox2026)
|
| 10 |
-
load_dotenv()
|
| 11 |
-
|
| 12 |
-
def test_deployment_flow():
|
| 13 |
-
"""Test the complete deployment flow"""
|
| 14 |
-
|
| 15 |
-
print("🧪 Testing ThoughtSpot Deployment Flow")
|
| 16 |
-
print("=" * 50)
|
| 17 |
-
|
| 18 |
-
# Check environment variables
|
| 19 |
-
required_vars = [
|
| 20 |
-
'THOUGHTSPOT_URL', 'THOUGHTSPOT_USERNAME', 'THOUGHTSPOT_PASSWORD',
|
| 21 |
-
'SNOWFLAKE_ACCOUNT', 'SNOWFLAKE_USER', 'SNOWFLAKE_PASSWORD',
|
| 22 |
-
'SNOWFLAKE_ROLE', 'SNOWFLAKE_WAREHOUSE'
|
| 23 |
-
]
|
| 24 |
-
|
| 25 |
-
missing_vars = []
|
| 26 |
-
for var in required_vars:
|
| 27 |
-
if not os.getenv(var):
|
| 28 |
-
missing_vars.append(var)
|
| 29 |
-
|
| 30 |
-
if missing_vars:
|
| 31 |
-
print("❌ Missing environment variables:")
|
| 32 |
-
for var in missing_vars:
|
| 33 |
-
print(f" - {var}")
|
| 34 |
-
print("\n💡 Copy your .env file from sandbox2026 or set these variables")
|
| 35 |
-
return False
|
| 36 |
-
|
| 37 |
-
print("✅ All environment variables found")
|
| 38 |
-
|
| 39 |
-
# Show the intended flow
|
| 40 |
-
print("\n🚀 Deployment Flow:")
|
| 41 |
-
print("0️⃣ Grant schema permissions to se_role")
|
| 42 |
-
print(" GRANT USAGE ON DATABASE DEMOBUILD TO ROLE se_role")
|
| 43 |
-
print(" GRANT USAGE ON SCHEMA DEMOBUILD.{schema} TO ROLE se_role")
|
| 44 |
-
print(" GRANT SELECT ON ALL TABLES IN SCHEMA DEMOBUILD.{schema} TO ROLE se_role")
|
| 45 |
-
print()
|
| 46 |
-
print("1️⃣ Create ThoughtSpot connection")
|
| 47 |
-
print("2️⃣ Create tables from DDL")
|
| 48 |
-
print("3️⃣ Create model")
|
| 49 |
-
|
| 50 |
-
print("\n✅ Flow validated - ready for deployment!")
|
| 51 |
-
return True
|
| 52 |
-
|
| 53 |
-
if __name__ == "__main__":
|
| 54 |
-
test_deployment_flow()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_end_to_end_demo.py
DELETED
|
@@ -1,174 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
Test End-to-End Demo System
|
| 4 |
-
Complete deployment test for creating a full demo model
|
| 5 |
-
"""
|
| 6 |
-
|
| 7 |
-
import os
|
| 8 |
-
import sys
|
| 9 |
-
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
| 10 |
-
|
| 11 |
-
from datetime import datetime
|
| 12 |
-
from dotenv import load_dotenv
|
| 13 |
-
from thoughtspot_deployer import ThoughtSpotDeployer
|
| 14 |
-
|
| 15 |
-
load_dotenv()
|
| 16 |
-
|
| 17 |
-
def test_end_to_end_demo():
|
| 18 |
-
"""Test complete end-to-end demo deployment"""
|
| 19 |
-
|
| 20 |
-
print("🚀 END-TO-END DEMO SYSTEM DEPLOYMENT")
|
| 21 |
-
print("=" * 60)
|
| 22 |
-
print("🎯 Creating complete demo model with all tables and proper structure")
|
| 23 |
-
|
| 24 |
-
# Use the working schema
|
| 25 |
-
schema_name = "20250918_161144_THOUG_SAL"
|
| 26 |
-
database = "DEMOBUILD"
|
| 27 |
-
|
| 28 |
-
# Complete DDL for full e-commerce demo
|
| 29 |
-
demo_ddl = """
|
| 30 |
-
CREATE TABLE CUSTOMERS (
|
| 31 |
-
CUSTOMERID NUMBER(38,0) NOT NULL,
|
| 32 |
-
FIRSTNAME VARCHAR(100) NOT NULL,
|
| 33 |
-
LASTNAME VARCHAR(100) NOT NULL,
|
| 34 |
-
EMAIL VARCHAR(255) NOT NULL,
|
| 35 |
-
PHONE VARCHAR(20) NOT NULL,
|
| 36 |
-
REGISTRATIONDATE DATE NOT NULL
|
| 37 |
-
);
|
| 38 |
-
|
| 39 |
-
CREATE TABLE PRODUCTS (
|
| 40 |
-
PRODUCTID NUMBER(38,0) NOT NULL,
|
| 41 |
-
PRODUCTNAME VARCHAR(255) NOT NULL,
|
| 42 |
-
CATEGORY VARCHAR(100) NOT NULL,
|
| 43 |
-
PRICE NUMBER(10,2) NOT NULL,
|
| 44 |
-
STOCKQUANTITY NUMBER(38,0) NOT NULL
|
| 45 |
-
);
|
| 46 |
-
|
| 47 |
-
CREATE TABLE ORDERS (
|
| 48 |
-
ORDERID NUMBER(38,0) NOT NULL,
|
| 49 |
-
CUSTOMERID NUMBER(38,0) NOT NULL,
|
| 50 |
-
ORDERDATE TIMESTAMP_NTZ(9) NOT NULL,
|
| 51 |
-
TOTALAMOUNT NUMBER(10,2) NOT NULL
|
| 52 |
-
);
|
| 53 |
-
|
| 54 |
-
CREATE TABLE ORDERITEMS (
|
| 55 |
-
ORDERITEMID NUMBER(38,0) NOT NULL,
|
| 56 |
-
ORDERID NUMBER(38,0) NOT NULL,
|
| 57 |
-
PRODUCTID NUMBER(38,0) NOT NULL,
|
| 58 |
-
QUANTITY NUMBER(38,0) NOT NULL,
|
| 59 |
-
UNITPRICE NUMBER(10,2) NOT NULL
|
| 60 |
-
);
|
| 61 |
-
|
| 62 |
-
CREATE TABLE SALES (
|
| 63 |
-
SALEID NUMBER(38,0) NOT NULL,
|
| 64 |
-
ORDERID NUMBER(38,0) NOT NULL,
|
| 65 |
-
SALESREPID NUMBER(38,0) NOT NULL,
|
| 66 |
-
SALEDATE TIMESTAMP_NTZ(9) NOT NULL
|
| 67 |
-
);
|
| 68 |
-
|
| 69 |
-
CREATE TABLE SALESREPS (
|
| 70 |
-
SALESREPID NUMBER(38,0) NOT NULL,
|
| 71 |
-
FIRSTNAME VARCHAR(100) NOT NULL,
|
| 72 |
-
LASTNAME VARCHAR(100) NOT NULL,
|
| 73 |
-
EMAIL VARCHAR(255) NOT NULL,
|
| 74 |
-
PHONE VARCHAR(20) NOT NULL,
|
| 75 |
-
HIREDATE DATE NOT NULL
|
| 76 |
-
);
|
| 77 |
-
"""
|
| 78 |
-
|
| 79 |
-
print(f"📊 Deploying to schema: {schema_name}")
|
| 80 |
-
print(f"📊 Database: {database}")
|
| 81 |
-
print(f"📊 Tables: 6 tables (complete e-commerce model)")
|
| 82 |
-
print()
|
| 83 |
-
|
| 84 |
-
try:
|
| 85 |
-
# Initialize deployer
|
| 86 |
-
deployer = ThoughtSpotDeployer()
|
| 87 |
-
|
| 88 |
-
# Create unique names for this demo
|
| 89 |
-
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
| 90 |
-
connection_name = f"demo_system_{timestamp}"
|
| 91 |
-
|
| 92 |
-
print(f"🔗 Demo Connection: {connection_name}")
|
| 93 |
-
print(f"🏷️ This will be your complete demo system!")
|
| 94 |
-
print()
|
| 95 |
-
|
| 96 |
-
# Deploy everything using the updated deployer
|
| 97 |
-
print("🚀 Starting end-to-end deployment...")
|
| 98 |
-
results = deployer.deploy_all(
|
| 99 |
-
ddl=demo_ddl,
|
| 100 |
-
database=database,
|
| 101 |
-
schema=schema_name,
|
| 102 |
-
connection_name=connection_name
|
| 103 |
-
)
|
| 104 |
-
|
| 105 |
-
print("\n" + "="*70)
|
| 106 |
-
print("📋 END-TO-END DEPLOYMENT RESULTS")
|
| 107 |
-
print("="*70)
|
| 108 |
-
|
| 109 |
-
if results['success']:
|
| 110 |
-
print("🎉 DEPLOYMENT SUCCESSFUL!")
|
| 111 |
-
print(f"✅ Connection: {results['connection']}")
|
| 112 |
-
print(f"✅ Tables created: {len(results['tables'])}")
|
| 113 |
-
for table in results['tables']:
|
| 114 |
-
print(f" • {table}")
|
| 115 |
-
print(f"✅ Model: {results['model']}")
|
| 116 |
-
|
| 117 |
-
print("\n🎯 YOUR DEMO SYSTEM IS READY!")
|
| 118 |
-
print("="*50)
|
| 119 |
-
print(f"🔗 Connection Name: '{results['connection']}'")
|
| 120 |
-
print(f"📊 Model Name: '{results['model']}'")
|
| 121 |
-
print()
|
| 122 |
-
print("✨ WHAT YOU CAN DO NOW:")
|
| 123 |
-
print("1. 📈 Create searches and answers using the model")
|
| 124 |
-
print("2. 🎨 Build dashboards with the data")
|
| 125 |
-
print("3. 🤖 Use the model for AI-powered analytics")
|
| 126 |
-
print("4. 👥 Share with demo users")
|
| 127 |
-
print()
|
| 128 |
-
print("🔍 MODEL FEATURES:")
|
| 129 |
-
print("• ✅ No ID columns cluttering the interface")
|
| 130 |
-
print("• ✅ Smart column naming (RepFirstname, repPhone, etc.)")
|
| 131 |
-
print("• ✅ Proper measures vs attributes")
|
| 132 |
-
print("• ✅ Clean, business-friendly structure")
|
| 133 |
-
print("• ✅ Matches your proven boone_test5 pattern")
|
| 134 |
-
|
| 135 |
-
else:
|
| 136 |
-
print("❌ DEPLOYMENT FAILED")
|
| 137 |
-
if results['errors']:
|
| 138 |
-
print("Errors encountered:")
|
| 139 |
-
for i, error in enumerate(results['errors'], 1):
|
| 140 |
-
print(f" {i}. {error}")
|
| 141 |
-
|
| 142 |
-
print("\n🔧 TROUBLESHOOTING:")
|
| 143 |
-
print("• Check ThoughtSpot connection settings")
|
| 144 |
-
print("• Verify Snowflake schema exists and is accessible")
|
| 145 |
-
print("• Ensure proper permissions are set")
|
| 146 |
-
|
| 147 |
-
return results['success']
|
| 148 |
-
|
| 149 |
-
except Exception as e:
|
| 150 |
-
print(f"\n❌ Deployment failed with error: {e}")
|
| 151 |
-
import traceback
|
| 152 |
-
traceback.print_exc()
|
| 153 |
-
|
| 154 |
-
print("\n🔧 COMMON ISSUES:")
|
| 155 |
-
print("• Authentication failure - check credentials")
|
| 156 |
-
print("• Schema not found - run demo_prep.py first")
|
| 157 |
-
print("• Permission issues - verify Snowflake roles")
|
| 158 |
-
|
| 159 |
-
return False
|
| 160 |
-
|
| 161 |
-
if __name__ == "__main__":
|
| 162 |
-
print("🎬 STARTING END-TO-END DEMO SYSTEM DEPLOYMENT")
|
| 163 |
-
print("This will create a complete ThoughtSpot demo environment!")
|
| 164 |
-
print()
|
| 165 |
-
|
| 166 |
-
success = test_end_to_end_demo()
|
| 167 |
-
|
| 168 |
-
if success:
|
| 169 |
-
print(f"\n🎉 SUCCESS! Your end-to-end demo system is ready to use!")
|
| 170 |
-
print("Check ThoughtSpot for your new connection and model.")
|
| 171 |
-
else:
|
| 172 |
-
print(f"\n❌ FAILED! Please check the error messages above.")
|
| 173 |
-
|
| 174 |
-
sys.exit(0 if success else 1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
tests/test_framework.py
DELETED
|
@@ -1,447 +0,0 @@
|
|
| 1 |
-
#!/usr/bin/env python3
|
| 2 |
-
"""
|
| 3 |
-
Enhanced Test Framework for Demo Wire
|
| 4 |
-
Comprehensive testing with schema validation, performance metrics, and robust reporting
|
| 5 |
-
"""
|
| 6 |
-
|
| 7 |
-
import os
|
| 8 |
-
import sys
|
| 9 |
-
import time
|
| 10 |
-
import json
|
| 11 |
-
import traceback
|
| 12 |
-
from datetime import datetime
|
| 13 |
-
from typing import Dict, List, Any, Optional, Tuple
|
| 14 |
-
from dataclasses import dataclass, asdict
|
| 15 |
-
from enum import Enum
|
| 16 |
-
|
| 17 |
-
# Add parent directory to path
|
| 18 |
-
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
| 19 |
-
|
| 20 |
-
from dotenv import load_dotenv
|
| 21 |
-
load_dotenv()
|
| 22 |
-
|
| 23 |
-
class TestResult(Enum):
|
| 24 |
-
PASS = "✅ PASS"
|
| 25 |
-
FAIL = "❌ FAIL"
|
| 26 |
-
SKIP = "⏭️ SKIP"
|
| 27 |
-
WARN = "⚠️ WARN"
|
| 28 |
-
|
| 29 |
-
@dataclass
|
| 30 |
-
class TestCase:
|
| 31 |
-
"""Individual test case with metadata"""
|
| 32 |
-
name: str
|
| 33 |
-
description: str
|
| 34 |
-
category: str
|
| 35 |
-
priority: int # 1=critical, 2=important, 3=nice-to-have
|
| 36 |
-
timeout: int = 60 # seconds
|
| 37 |
-
prerequisites: List[str] = None
|
| 38 |
-
expected_duration: float = 0.0 # seconds
|
| 39 |
-
|
| 40 |
-
def __post_init__(self):
|
| 41 |
-
if self.prerequisites is None:
|
| 42 |
-
self.prerequisites = []
|
| 43 |
-
|
| 44 |
-
@dataclass
|
| 45 |
-
class TestReport:
|
| 46 |
-
"""Test execution report"""
|
| 47 |
-
test_name: str
|
| 48 |
-
result: TestResult
|
| 49 |
-
duration: float
|
| 50 |
-
error_message: str = ""
|
| 51 |
-
details: Dict[str, Any] = None
|
| 52 |
-
timestamp: str = ""
|
| 53 |
-
|
| 54 |
-
def __post_init__(self):
|
| 55 |
-
if self.details is None:
|
| 56 |
-
self.details = {}
|
| 57 |
-
if not self.timestamp:
|
| 58 |
-
self.timestamp = datetime.now().isoformat()
|
| 59 |
-
|
| 60 |
-
class TestFramework:
|
| 61 |
-
"""Enhanced test framework with comprehensive reporting and validation"""
|
| 62 |
-
|
| 63 |
-
def __init__(self):
|
| 64 |
-
self.results: List[TestReport] = []
|
| 65 |
-
self.start_time = None
|
| 66 |
-
self.test_cases: List[TestCase] = []
|
| 67 |
-
self.setup_complete = False
|
| 68 |
-
|
| 69 |
-
def add_test(self, test_case: TestCase):
|
| 70 |
-
"""Register a test case"""
|
| 71 |
-
self.test_cases.append(test_case)
|
| 72 |
-
|
| 73 |
-
def validate_environment(self) -> TestReport:
|
| 74 |
-
"""Validate required environment variables and dependencies"""
|
| 75 |
-
start_time = time.time()
|
| 76 |
-
|
| 77 |
-
required_vars = [
|
| 78 |
-
'THOUGHTSPOT_URL', 'THOUGHTSPOT_USERNAME', 'THOUGHTSPOT_PASSWORD',
|
| 79 |
-
'SNOWFLAKE_ACCOUNT', 'SNOWFLAKE_USER', 'SNOWFLAKE_PRIVATE_KEY',
|
| 80 |
-
'SNOWFLAKE_ROLE', 'SNOWFLAKE_WAREHOUSE'
|
| 81 |
-
]
|
| 82 |
-
|
| 83 |
-
missing_vars = []
|
| 84 |
-
for var in required_vars:
|
| 85 |
-
if not os.getenv(var):
|
| 86 |
-
missing_vars.append(var)
|
| 87 |
-
|
| 88 |
-
duration = time.time() - start_time
|
| 89 |
-
|
| 90 |
-
if missing_vars:
|
| 91 |
-
return TestReport(
|
| 92 |
-
test_name="Environment Validation",
|
| 93 |
-
result=TestResult.FAIL,
|
| 94 |
-
duration=duration,
|
| 95 |
-
error_message=f"Missing environment variables: {', '.join(missing_vars)}",
|
| 96 |
-
details={"missing_vars": missing_vars}
|
| 97 |
-
)
|
| 98 |
-
|
| 99 |
-
return TestReport(
|
| 100 |
-
test_name="Environment Validation",
|
| 101 |
-
result=TestResult.PASS,
|
| 102 |
-
duration=duration,
|
| 103 |
-
details={"validated_vars": len(required_vars)}
|
| 104 |
-
)
|
| 105 |
-
|
| 106 |
-
def validate_schema_structure(self, ddl: str) -> TestReport:
|
| 107 |
-
"""Validate DDL structure and syntax"""
|
| 108 |
-
start_time = time.time()
|
| 109 |
-
|
| 110 |
-
try:
|
| 111 |
-
# Basic DDL validation
|
| 112 |
-
validation_results = {
|
| 113 |
-
"has_create_statements": "CREATE TABLE" in ddl.upper(),
|
| 114 |
-
"table_count": ddl.upper().count("CREATE TABLE"),
|
| 115 |
-
"has_primary_keys": "PRIMARY KEY" in ddl.upper(),
|
| 116 |
-
"has_not_null": "NOT NULL" in ddl.upper(),
|
| 117 |
-
"estimated_complexity": len(ddl.split('\n'))
|
| 118 |
-
}
|
| 119 |
-
|
| 120 |
-
# Check for common DDL patterns
|
| 121 |
-
critical_issues = []
|
| 122 |
-
warnings = []
|
| 123 |
-
|
| 124 |
-
if validation_results["table_count"] == 0:
|
| 125 |
-
critical_issues.append("No CREATE TABLE statements found")
|
| 126 |
-
|
| 127 |
-
if not validation_results["has_not_null"]:
|
| 128 |
-
warnings.append("No NOT NULL constraints found - consider data quality")
|
| 129 |
-
|
| 130 |
-
# Validate column naming patterns
|
| 131 |
-
lines = ddl.split('\n')
|
| 132 |
-
for line in lines:
|
| 133 |
-
if 'CREATE TABLE' in line.upper():
|
| 134 |
-
table_name = line.split()[-2] if '(' in line else line.split()[-1]
|
| 135 |
-
validation_results[f"table_{table_name}"] = {
|
| 136 |
-
"found": True,
|
| 137 |
-
"line": line.strip()
|
| 138 |
-
}
|
| 139 |
-
|
| 140 |
-
duration = time.time() - start_time
|
| 141 |
-
result = TestResult.FAIL if critical_issues else (TestResult.WARN if warnings else TestResult.PASS)
|
| 142 |
-
|
| 143 |
-
return TestReport(
|
| 144 |
-
test_name="Schema Structure Validation",
|
| 145 |
-
result=result,
|
| 146 |
-
duration=duration,
|
| 147 |
-
error_message="; ".join(critical_issues) if critical_issues else "",
|
| 148 |
-
details={
|
| 149 |
-
**validation_results,
|
| 150 |
-
"warnings": warnings,
|
| 151 |
-
"critical_issues": critical_issues
|
| 152 |
-
}
|
| 153 |
-
)
|
| 154 |
-
|
| 155 |
-
except Exception as e:
|
| 156 |
-
return TestReport(
|
| 157 |
-
test_name="Schema Structure Validation",
|
| 158 |
-
result=TestResult.FAIL,
|
| 159 |
-
duration=time.time() - start_time,
|
| 160 |
-
error_message=f"Validation failed: {str(e)}"
|
| 161 |
-
)
|
| 162 |
-
|
| 163 |
-
def test_thoughtspot_auth(self) -> TestReport:
|
| 164 |
-
"""Test ThoughtSpot authentication"""
|
| 165 |
-
start_time = time.time()
|
| 166 |
-
|
| 167 |
-
try:
|
| 168 |
-
from thoughtspot_deployer import ThoughtSpotDeployer
|
| 169 |
-
|
| 170 |
-
deployer = ThoughtSpotDeployer()
|
| 171 |
-
auth_success = deployer.authenticate()
|
| 172 |
-
|
| 173 |
-
duration = time.time() - start_time
|
| 174 |
-
|
| 175 |
-
return TestReport(
|
| 176 |
-
test_name="ThoughtSpot Authentication",
|
| 177 |
-
result=TestResult.PASS if auth_success else TestResult.FAIL,
|
| 178 |
-
duration=duration,
|
| 179 |
-
error_message="" if auth_success else "Authentication failed",
|
| 180 |
-
details={
|
| 181 |
-
"auth_method": "session_based",
|
| 182 |
-
"response_time": duration
|
| 183 |
-
}
|
| 184 |
-
)
|
| 185 |
-
|
| 186 |
-
except Exception as e:
|
| 187 |
-
return TestReport(
|
| 188 |
-
test_name="ThoughtSpot Authentication",
|
| 189 |
-
result=TestResult.FAIL,
|
| 190 |
-
duration=time.time() - start_time,
|
| 191 |
-
error_message=f"Auth test failed: {str(e)}"
|
| 192 |
-
)
|
| 193 |
-
|
| 194 |
-
def test_snowflake_connection(self) -> TestReport:
|
| 195 |
-
"""Test Snowflake key pair authentication"""
|
| 196 |
-
start_time = time.time()
|
| 197 |
-
|
| 198 |
-
try:
|
| 199 |
-
import snowflake.connector
|
| 200 |
-
from snowflake_auth import get_snowflake_connection_params
|
| 201 |
-
|
| 202 |
-
params = get_snowflake_connection_params()
|
| 203 |
-
conn = snowflake.connector.connect(**params)
|
| 204 |
-
cursor = conn.cursor()
|
| 205 |
-
|
| 206 |
-
# Test basic query
|
| 207 |
-
cursor.execute("SELECT CURRENT_VERSION()")
|
| 208 |
-
version = cursor.fetchone()[0]
|
| 209 |
-
|
| 210 |
-
conn.close()
|
| 211 |
-
duration = time.time() - start_time
|
| 212 |
-
|
| 213 |
-
return TestReport(
|
| 214 |
-
test_name="Snowflake Connection",
|
| 215 |
-
result=TestResult.PASS,
|
| 216 |
-
duration=duration,
|
| 217 |
-
details={
|
| 218 |
-
"snowflake_version": version,
|
| 219 |
-
"auth_method": "key_pair",
|
| 220 |
-
"connection_time": duration
|
| 221 |
-
}
|
| 222 |
-
)
|
| 223 |
-
|
| 224 |
-
except Exception as e:
|
| 225 |
-
return TestReport(
|
| 226 |
-
test_name="Snowflake Connection",
|
| 227 |
-
result=TestResult.FAIL,
|
| 228 |
-
duration=time.time() - start_time,
|
| 229 |
-
error_message=f"Snowflake connection failed: {str(e)}"
|
| 230 |
-
)
|
| 231 |
-
|
| 232 |
-
def test_deployment_pipeline(self, ddl: str, database: str, schema: str) -> TestReport:
|
| 233 |
-
"""Test complete deployment pipeline"""
|
| 234 |
-
start_time = time.time()
|
| 235 |
-
|
| 236 |
-
try:
|
| 237 |
-
from thoughtspot_deployer import ThoughtSpotDeployer
|
| 238 |
-
|
| 239 |
-
deployer = ThoughtSpotDeployer()
|
| 240 |
-
|
| 241 |
-
# Authenticate first
|
| 242 |
-
if not deployer.authenticate():
|
| 243 |
-
return TestReport(
|
| 244 |
-
test_name="Deployment Pipeline",
|
| 245 |
-
result=TestResult.FAIL,
|
| 246 |
-
duration=time.time() - start_time,
|
| 247 |
-
error_message="Authentication failed"
|
| 248 |
-
)
|
| 249 |
-
|
| 250 |
-
# Run deployment
|
| 251 |
-
results = deployer.deploy_all(
|
| 252 |
-
ddl=ddl,
|
| 253 |
-
database=database,
|
| 254 |
-
schema=schema
|
| 255 |
-
)
|
| 256 |
-
|
| 257 |
-
duration = time.time() - start_time
|
| 258 |
-
|
| 259 |
-
# Analyze results
|
| 260 |
-
pipeline_success = {
|
| 261 |
-
"connection": bool(results.get('connection')),
|
| 262 |
-
"tables": len(results.get('tables', [])) > 0,
|
| 263 |
-
"worksheet": bool(results.get('worksheet')),
|
| 264 |
-
"errors": results.get('errors', [])
|
| 265 |
-
}
|
| 266 |
-
|
| 267 |
-
overall_success = pipeline_success["connection"] and pipeline_success["tables"]
|
| 268 |
-
|
| 269 |
-
return TestReport(
|
| 270 |
-
test_name="Deployment Pipeline",
|
| 271 |
-
result=TestResult.PASS if overall_success else TestResult.FAIL,
|
| 272 |
-
duration=duration,
|
| 273 |
-
error_message=f"Errors: {len(pipeline_success['errors'])}" if pipeline_success['errors'] else "",
|
| 274 |
-
details={
|
| 275 |
-
**pipeline_success,
|
| 276 |
-
"deployment_time": duration,
|
| 277 |
-
"schema_used": schema,
|
| 278 |
-
"tables_created": len(results.get('tables', []))
|
| 279 |
-
}
|
| 280 |
-
)
|
| 281 |
-
|
| 282 |
-
except Exception as e:
|
| 283 |
-
return TestReport(
|
| 284 |
-
test_name="Deployment Pipeline",
|
| 285 |
-
result=TestResult.FAIL,
|
| 286 |
-
duration=time.time() - start_time,
|
| 287 |
-
error_message=f"Pipeline test failed: {str(e)}"
|
| 288 |
-
)
|
| 289 |
-
|
| 290 |
-
def run_performance_test(self, test_func, *args, **kwargs) -> Tuple[Any, float, bool]:
|
| 291 |
-
"""Run a test function with performance monitoring"""
|
| 292 |
-
start_time = time.time()
|
| 293 |
-
|
| 294 |
-
try:
|
| 295 |
-
result = test_func(*args, **kwargs)
|
| 296 |
-
duration = time.time() - start_time
|
| 297 |
-
return result, duration, True
|
| 298 |
-
except Exception as e:
|
| 299 |
-
duration = time.time() - start_time
|
| 300 |
-
print(f"⚠️ Performance test failed: {e}")
|
| 301 |
-
return None, duration, False
|
| 302 |
-
|
| 303 |
-
def run_all_tests(self, ddl: str = None, database: str = "DEMOBUILD",
|
| 304 |
-
schema: str = None) -> Dict[str, Any]:
|
| 305 |
-
"""Run comprehensive test suite"""
|
| 306 |
-
|
| 307 |
-
print("🧪 ENHANCED TEST FRAMEWORK")
|
| 308 |
-
print("=" * 60)
|
| 309 |
-
print(f"🚀 Starting comprehensive test suite at {datetime.now()}")
|
| 310 |
-
|
| 311 |
-
self.start_time = time.time()
|
| 312 |
-
|
| 313 |
-
# Auto-detect schema if not provided
|
| 314 |
-
if not schema:
|
| 315 |
-
# Look for recent schemas
|
| 316 |
-
schema_pattern = datetime.now().strftime("2025%m%d")
|
| 317 |
-
schema = f"{schema_pattern}_ENHANCED_TEST"
|
| 318 |
-
print(f"📊 Using auto-generated schema: {schema}")
|
| 319 |
-
|
| 320 |
-
# Default DDL for testing
|
| 321 |
-
if not ddl:
|
| 322 |
-
ddl = """
|
| 323 |
-
CREATE TABLE CUSTOMERS (
|
| 324 |
-
CUSTOMERID NUMBER(38,0) NOT NULL,
|
| 325 |
-
FIRSTNAME VARCHAR(50) NOT NULL,
|
| 326 |
-
LASTNAME VARCHAR(50) NOT NULL,
|
| 327 |
-
EMAIL VARCHAR(100) NOT NULL
|
| 328 |
-
);
|
| 329 |
-
|
| 330 |
-
CREATE TABLE PRODUCTS (
|
| 331 |
-
PRODUCTID NUMBER(38,0) NOT NULL,
|
| 332 |
-
PRODUCTNAME VARCHAR(100) NOT NULL,
|
| 333 |
-
PRICE NUMBER(10,2) NOT NULL
|
| 334 |
-
);
|
| 335 |
-
"""
|
| 336 |
-
|
| 337 |
-
# Run tests in order
|
| 338 |
-
tests_to_run = [
|
| 339 |
-
("Environment", lambda: self.validate_environment()),
|
| 340 |
-
("Schema Validation", lambda: self.validate_schema_structure(ddl)),
|
| 341 |
-
("Snowflake Connection", lambda: self.test_snowflake_connection()),
|
| 342 |
-
("ThoughtSpot Auth", lambda: self.test_thoughtspot_auth()),
|
| 343 |
-
("Full Deployment", lambda: self.test_deployment_pipeline(ddl, database, schema))
|
| 344 |
-
]
|
| 345 |
-
|
| 346 |
-
print("\n📋 Running Test Suite:")
|
| 347 |
-
|
| 348 |
-
for test_name, test_func in tests_to_run:
|
| 349 |
-
print(f"\n🔄 Running: {test_name}...")
|
| 350 |
-
|
| 351 |
-
try:
|
| 352 |
-
report = test_func()
|
| 353 |
-
self.results.append(report)
|
| 354 |
-
|
| 355 |
-
# Show immediate feedback
|
| 356 |
-
duration_str = f"{report.duration:.2f}s"
|
| 357 |
-
print(f" {report.result.value} {test_name} ({duration_str})")
|
| 358 |
-
|
| 359 |
-
if report.error_message:
|
| 360 |
-
print(f" 💬 {report.error_message}")
|
| 361 |
-
|
| 362 |
-
except Exception as e:
|
| 363 |
-
error_report = TestReport(
|
| 364 |
-
test_name=test_name,
|
| 365 |
-
result=TestResult.FAIL,
|
| 366 |
-
duration=0.0,
|
| 367 |
-
error_message=f"Test execution failed: {str(e)}"
|
| 368 |
-
)
|
| 369 |
-
self.results.append(error_report)
|
| 370 |
-
print(f" ❌ FAIL {test_name} (execution error)")
|
| 371 |
-
|
| 372 |
-
# Generate comprehensive report
|
| 373 |
-
return self.generate_report()
|
| 374 |
-
|
| 375 |
-
def generate_report(self) -> Dict[str, Any]:
|
| 376 |
-
"""Generate comprehensive test report"""
|
| 377 |
-
|
| 378 |
-
total_duration = time.time() - self.start_time if self.start_time else 0
|
| 379 |
-
|
| 380 |
-
# Calculate statistics
|
| 381 |
-
stats = {
|
| 382 |
-
"total_tests": len(self.results),
|
| 383 |
-
"passed": len([r for r in self.results if r.result == TestResult.PASS]),
|
| 384 |
-
"failed": len([r for r in self.results if r.result == TestResult.FAIL]),
|
| 385 |
-
"warnings": len([r for r in self.results if r.result == TestResult.WARN]),
|
| 386 |
-
"skipped": len([r for r in self.results if r.result == TestResult.SKIP]),
|
| 387 |
-
"total_duration": total_duration,
|
| 388 |
-
"average_duration": total_duration / len(self.results) if self.results else 0
|
| 389 |
-
}
|
| 390 |
-
|
| 391 |
-
success_rate = (stats["passed"] / stats["total_tests"] * 100) if stats["total_tests"] > 0 else 0
|
| 392 |
-
|
| 393 |
-
# Print summary report
|
| 394 |
-
print("\n" + "=" * 60)
|
| 395 |
-
print("📊 TEST SUMMARY REPORT")
|
| 396 |
-
print("=" * 60)
|
| 397 |
-
|
| 398 |
-
print(f"🎯 Overall Success Rate: {success_rate:.1f}%")
|
| 399 |
-
print(f"⏱️ Total Duration: {total_duration:.2f}s")
|
| 400 |
-
print(f"📈 Tests: {stats['passed']}✅ {stats['failed']}❌ {stats['warnings']}⚠️ {stats['skipped']}⏭️")
|
| 401 |
-
|
| 402 |
-
print(f"\n📋 Individual Results:")
|
| 403 |
-
for result in self.results:
|
| 404 |
-
duration_str = f"({result.duration:.2f}s)"
|
| 405 |
-
print(f" {result.result.value} {result.test_name} {duration_str}")
|
| 406 |
-
if result.error_message:
|
| 407 |
-
print(f" 💬 {result.error_message}")
|
| 408 |
-
|
| 409 |
-
# Recommendations
|
| 410 |
-
print(f"\n🔧 RECOMMENDATIONS:")
|
| 411 |
-
if stats["failed"] > 0:
|
| 412 |
-
print(f" • Fix {stats['failed']} failing tests before deployment")
|
| 413 |
-
if stats["warnings"] > 0:
|
| 414 |
-
print(f" • Review {stats['warnings']} warnings for potential issues")
|
| 415 |
-
if success_rate == 100:
|
| 416 |
-
print(f" 🎉 All tests passing! System ready for deployment!")
|
| 417 |
-
|
| 418 |
-
print("=" * 60)
|
| 419 |
-
|
| 420 |
-
# Save detailed report
|
| 421 |
-
report_data = {
|
| 422 |
-
"timestamp": datetime.now().isoformat(),
|
| 423 |
-
"statistics": stats,
|
| 424 |
-
"success_rate": success_rate,
|
| 425 |
-
"results": [asdict(r) for r in self.results]
|
| 426 |
-
}
|
| 427 |
-
|
| 428 |
-
# Save to file
|
| 429 |
-
report_file = f"test_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
| 430 |
-
with open(os.path.join("tests", report_file), "w") as f:
|
| 431 |
-
json.dump(report_data, f, indent=2)
|
| 432 |
-
|
| 433 |
-
print(f"📄 Detailed report saved: tests/{report_file}")
|
| 434 |
-
|
| 435 |
-
return report_data
|
| 436 |
-
|
| 437 |
-
def main():
|
| 438 |
-
"""Run enhanced test framework"""
|
| 439 |
-
framework = TestFramework()
|
| 440 |
-
|
| 441 |
-
# Example usage - can be customized
|
| 442 |
-
results = framework.run_all_tests()
|
| 443 |
-
|
| 444 |
-
return results["success_rate"] == 100.0
|
| 445 |
-
|
| 446 |
-
if __name__ == "__main__":
|
| 447 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|