mikeboone commited on
Commit
a2a4e6d
·
2 Parent(s): b91d31bb2a3e22

Merge feature/naming-convention-fix into feature/chat-interface

Browse files
CLAUDE.md CHANGED
@@ -213,4 +213,4 @@ This software is curretnly stored in my repo but will be open sourced and move t
213
  ---
214
 
215
  *Last Updated: December 10, 2025*
216
- *This is a living document - update as you learn more about the project*
 
213
  ---
214
 
215
  *Last Updated: December 10, 2025*
216
+ *This is a living document - update as you learn more about the project*
cdw_connector.py CHANGED
@@ -72,19 +72,40 @@ class SnowflakeDeployer:
72
  except Exception as e:
73
  print(f"Error closing connection: {str(e)}")
74
 
75
- def create_demo_schema_and_deploy(self, company_name: str, use_case: str, ddl_statements: str) -> Tuple[bool, Optional[str], str]:
76
- """Create timestamped schema and deploy DDL statements"""
 
 
 
 
 
77
  if not self.connection:
78
  success, message = self.connect()
79
  if not success:
80
  return False, None, f"Connection failed: {message}"
81
-
82
  try:
83
- # Generate schema name with timestamp
84
- timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
85
- company_short = company_name.replace(" ", "")[:5] # First 5 chars
86
- use_case_short = use_case.replace(" ", "")[:3] # First 3 chars
87
- schema_name = f"{timestamp}_{company_short}_{use_case_short}".upper()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
 
89
  cursor = self.connection.cursor()
90
 
@@ -127,15 +148,20 @@ class SnowflakeDeployer:
127
  # Check autocommit setting
128
  print(f"🔍 DEBUG: Connection autocommit: {self.connection.autocommit}")
129
 
 
 
 
 
 
130
  # VERIFY the schema actually exists
131
  print(f"🔍 DEBUG: Verifying schema exists...")
132
  cursor.execute(f'SELECT CURRENT_SCHEMA()')
133
  current_schema = cursor.fetchone()[0]
134
  print(f"🔍 DEBUG: Current schema: {current_schema}")
135
  print(f"🔍 DEBUG: Schema verification completed - using current schema context")
136
-
137
  cursor.close()
138
-
139
  success_message = f"Schema '{schema_name}' created successfully with {executed_count} tables"
140
  return True, schema_name, success_message
141
 
 
72
  except Exception as e:
73
  print(f"Error closing connection: {str(e)}")
74
 
75
+ def create_demo_schema_and_deploy(self, company_name: str, use_case: str, ddl_statements: str, naming_prefix: str = "") -> Tuple[bool, Optional[str], str]:
76
+ """Create timestamped schema and deploy DDL statements
77
+
78
+ Args:
79
+ naming_prefix: Optional custom prefix for schema name
80
+ Format: {prefix}_{day+ms}_scm or just {day+ms}_scm if blank
81
+ """
82
  if not self.connection:
83
  success, message = self.connect()
84
  if not success:
85
  return False, None, f"Connection failed: {message}"
86
+
87
  try:
88
+ # Generate schema name using same convention as ThoughtSpot objects
89
+ import re
90
+ from datetime import datetime as dt
91
+ now = dt.now()
92
+
93
+ # Clean prefix (uppercase, alphanumeric only) - can be empty string
94
+ if naming_prefix:
95
+ prefix_clean = re.sub(r'[^a-zA-Z0-9]', '', naming_prefix.upper())
96
+ else:
97
+ prefix_clean = ""
98
+
99
+ # Short unique timestamp: day (2 digits) + microseconds (6 digits)
100
+ day = now.strftime('%d')
101
+ microsec = now.strftime('%f') # 6 digits
102
+ short_timestamp = f"{day}{microsec}"
103
+
104
+ # Base format: PREFIX_DDMICROS_scm (prefix can be empty)
105
+ if prefix_clean:
106
+ schema_name = f"{prefix_clean}_{short_timestamp}_scm"
107
+ else:
108
+ schema_name = f"{short_timestamp}_scm"
109
 
110
  cursor = self.connection.cursor()
111
 
 
148
  # Check autocommit setting
149
  print(f"🔍 DEBUG: Connection autocommit: {self.connection.autocommit}")
150
 
151
+ # COMMIT the transaction
152
+ print(f"🔍 DEBUG: Committing transaction...")
153
+ self.connection.commit()
154
+ print(f"✅ Transaction committed - {executed_count} tables created")
155
+
156
  # VERIFY the schema actually exists
157
  print(f"🔍 DEBUG: Verifying schema exists...")
158
  cursor.execute(f'SELECT CURRENT_SCHEMA()')
159
  current_schema = cursor.fetchone()[0]
160
  print(f"🔍 DEBUG: Current schema: {current_schema}")
161
  print(f"🔍 DEBUG: Schema verification completed - using current schema context")
162
+
163
  cursor.close()
164
+
165
  success_message = f"Schema '{schema_name}' created successfully with {executed_count} tables"
166
  return True, schema_name, success_message
167
 
demo_logger.py CHANGED
@@ -25,17 +25,24 @@ load_dotenv()
25
  class DemoLogger:
26
  """Comprehensive logger for demo creation process"""
27
 
28
- def __init__(self, session_id: str = None, user_email: str = None):
29
  """
30
  Initialize logger with session ID and optional user email
31
 
32
  Args:
33
  session_id: Unique ID for this demo creation session
34
  user_email: User's email for tracking
 
35
  """
36
  self.session_id = session_id or datetime.now().strftime("%Y%m%d_%H%M%S")
37
  self.user_email = user_email or "anonymous"
38
 
 
 
 
 
 
 
39
  # Create logs directory
40
  self.log_dir = Path("logs") / "demo_sessions" / self.session_id
41
  self.log_dir.mkdir(parents=True, exist_ok=True)
@@ -64,7 +71,14 @@ class DemoLogger:
64
  "status": "started"
65
  }
66
 
67
- self.logger.info(f"Demo session started: {self.session_id}")
 
 
 
 
 
 
 
68
 
69
  def setup_logger(self):
70
  """Set up Python logger"""
@@ -100,47 +114,79 @@ class DemoLogger:
100
  self.session_data["llm_model"] = llm_model
101
  self.session_data["settings"].update(kwargs)
102
 
103
- self.logger.info(f"Demo Configuration:")
104
- self.logger.info(f" Use Case: {use_case}")
105
- self.logger.info(f" Company: {company}")
106
- self.logger.info(f" URL: {company_url}")
107
- self.logger.info(f" Industry: {industry}")
108
- self.logger.info(f" LLM Model: {llm_model}")
 
 
 
 
 
 
 
 
 
 
 
 
 
109
 
110
  self.save_session_data()
111
 
112
  def log_ddl(self, ddl_content: str, schema_name: str = None):
113
- """Log the generated DDL"""
114
- self.session_data["ddl"] = ddl_content
 
 
 
 
 
115
  self.session_data["schema_name"] = schema_name
116
 
117
- # Save DDL to separate file
118
  ddl_file = self.log_dir / "generated_ddl.sql"
119
  with open(ddl_file, 'w') as f:
120
  f.write(ddl_content)
121
 
122
  # Log summary
123
  tables = ddl_content.count("CREATE TABLE")
124
- self.logger.info(f"DDL Generated: {tables} tables, {len(ddl_content)} characters")
125
- self.logger.info(f"DDL saved to: {ddl_file}")
126
-
127
- # Check for SALESTRANSACTIONS table (known issue)
128
- if "SALESTRANSACTIONS" not in ddl_content.upper():
129
- self.log_warning("SALESTRANSACTIONS table not found in DDL - this will cause deployment issues!")
 
 
 
 
 
 
 
 
 
130
 
131
  self.save_session_data()
132
 
133
  def log_population_code(self, code: str):
134
- """Log the generated population code"""
135
- self.session_data["population_code"] = code[:5000] # Truncate for storage
 
136
 
137
- # Save full code to file
138
  code_file = self.log_dir / "population_code.py"
139
  with open(code_file, 'w') as f:
140
  f.write(code)
141
 
142
- self.logger.info(f"Population code generated: {len(code)} characters")
143
- self.logger.info(f"Code saved to: {code_file}")
 
 
 
 
144
 
145
  self.save_session_data()
146
 
@@ -148,9 +194,16 @@ class DemoLogger:
148
  """Log a discovered outlier"""
149
  self.session_data["outliers"].append(outlier)
150
 
151
- self.logger.info(f"Outlier discovered: {outlier.get('title', 'Unknown')}")
152
- self.logger.debug(f" Insight: {outlier.get('insight', 'N/A')}")
153
- self.logger.debug(f" Impact: {outlier.get('impact', 'N/A')}")
 
 
 
 
 
 
 
154
 
155
  self.save_session_data()
156
 
@@ -190,13 +243,14 @@ class DemoLogger:
190
  self.save_session_data()
191
 
192
  def log_stage_completion(self, stage: str, success: bool,
193
- duration_seconds: float = None):
194
  """Log completion of a demo creation stage"""
195
  stage_data = {
196
  "stage": stage,
197
  "success": success,
198
  "duration": duration_seconds,
199
- "timestamp": datetime.now().isoformat()
 
200
  }
201
 
202
  if "stages" not in self.session_data:
@@ -205,9 +259,13 @@ class DemoLogger:
205
  self.session_data["stages"].append(stage_data)
206
 
207
  status = "✅ SUCCESS" if success else "❌ FAILED"
208
- self.logger.info(f"Stage '{stage}' {status}")
 
209
  if duration_seconds:
210
- self.logger.info(f" Duration: {duration_seconds:.2f} seconds")
 
 
 
211
 
212
  self.save_session_data()
213
 
 
25
  class DemoLogger:
26
  """Comprehensive logger for demo creation process"""
27
 
28
+ def __init__(self, session_id: str = None, user_email: str = None, debug_mode: bool = None):
29
  """
30
  Initialize logger with session ID and optional user email
31
 
32
  Args:
33
  session_id: Unique ID for this demo creation session
34
  user_email: User's email for tracking
35
+ debug_mode: Enable debug mode (saves DDL, population code, outliers)
36
  """
37
  self.session_id = session_id or datetime.now().strftime("%Y%m%d_%H%M%S")
38
  self.user_email = user_email or "anonymous"
39
 
40
+ # Check for DEBUG environment variable or parameter
41
+ if debug_mode is None:
42
+ self.debug_mode = os.getenv('DEBUG', 'false').lower() in ['true', '1', 'yes']
43
+ else:
44
+ self.debug_mode = debug_mode
45
+
46
  # Create logs directory
47
  self.log_dir = Path("logs") / "demo_sessions" / self.session_id
48
  self.log_dir.mkdir(parents=True, exist_ok=True)
 
71
  "status": "started"
72
  }
73
 
74
+ # Log session start with timestamp
75
+ self.logger.info("="*80)
76
+ self.logger.info(f"🚀 Demo Session Started")
77
+ self.logger.info(f" Session ID: {self.session_id}")
78
+ self.logger.info(f" User: {self.user_email}")
79
+ self.logger.info(f" Timestamp: {self.session_data['start_time']}")
80
+ self.logger.info(f" Debug Mode: {'ON' if self.debug_mode else 'OFF'}")
81
+ self.logger.info("="*80)
82
 
83
  def setup_logger(self):
84
  """Set up Python logger"""
 
114
  self.session_data["llm_model"] = llm_model
115
  self.session_data["settings"].update(kwargs)
116
 
117
+ self.logger.info("="*80)
118
+ self.logger.info("📋 Demo Configuration")
119
+ self.logger.info("="*80)
120
+ self.logger.info(f"🎯 MAIN SETTINGS:")
121
+ self.logger.info(f" Company URL: {company_url}")
122
+ self.logger.info(f" Use Case: {use_case}")
123
+ self.logger.info(f" LLM Model: {llm_model}")
124
+ self.logger.info(f"")
125
+ self.logger.info(f"📊 Additional Info:")
126
+ self.logger.info(f" Company: {company}")
127
+ self.logger.info(f" Industry: {industry}")
128
+
129
+ if kwargs:
130
+ self.logger.info(f"")
131
+ self.logger.info(f"⚙️ Other Settings:")
132
+ for key, value in kwargs.items():
133
+ self.logger.info(f" {key}: {value}")
134
+
135
+ self.logger.info("="*80)
136
 
137
  self.save_session_data()
138
 
139
  def log_ddl(self, ddl_content: str, schema_name: str = None):
140
+ """Log the generated DDL - ALWAYS saves to file"""
141
+ # Always save full DDL if debug mode
142
+ if self.debug_mode:
143
+ self.session_data["ddl"] = ddl_content
144
+ else:
145
+ self.session_data["ddl"] = ddl_content[:1000] + "..." if len(ddl_content) > 1000 else ddl_content
146
+
147
  self.session_data["schema_name"] = schema_name
148
 
149
+ # ALWAYS save DDL to separate file
150
  ddl_file = self.log_dir / "generated_ddl.sql"
151
  with open(ddl_file, 'w') as f:
152
  f.write(ddl_content)
153
 
154
  # Log summary
155
  tables = ddl_content.count("CREATE TABLE")
156
+ self.logger.info("="*80)
157
+ self.logger.info(f"📝 DDL Generated")
158
+ self.logger.info("="*80)
159
+ self.logger.info(f" Tables: {tables}")
160
+ self.logger.info(f" Size: {len(ddl_content)} characters")
161
+ self.logger.info(f" Schema: {schema_name or 'N/A'}")
162
+ self.logger.info(f" Saved to: {ddl_file}")
163
+
164
+ # Check for SALESTRANSACTIONS or SALES_TRANSACTIONS table (known issue)
165
+ ddl_upper = ddl_content.upper()
166
+ if "SALESTRANSACTIONS" not in ddl_upper and "SALES_TRANSACTIONS" not in ddl_upper:
167
+ self.logger.warning("⚠️ SALESTRANSACTIONS table not found in DDL - this will cause deployment issues!")
168
+ self.log_warning("SALESTRANSACTIONS table not found in DDL")
169
+
170
+ self.logger.info("="*80)
171
 
172
  self.save_session_data()
173
 
174
  def log_population_code(self, code: str):
175
+ """Log the generated population code - saves to file if DEBUG mode"""
176
+ # Always save summary
177
+ self.session_data["population_code"] = code[:5000] if not self.debug_mode else code
178
 
179
+ # ALWAYS save full code to file
180
  code_file = self.log_dir / "population_code.py"
181
  with open(code_file, 'w') as f:
182
  f.write(code)
183
 
184
+ self.logger.info("="*80)
185
+ self.logger.info(f"🐍 Population Code Generated")
186
+ self.logger.info("="*80)
187
+ self.logger.info(f" Size: {len(code)} characters")
188
+ self.logger.info(f" Saved to: {code_file}")
189
+ self.logger.info("="*80)
190
 
191
  self.save_session_data()
192
 
 
194
  """Log a discovered outlier"""
195
  self.session_data["outliers"].append(outlier)
196
 
197
+ self.logger.info(f"💡 Outlier discovered: {outlier.get('title', 'Unknown')}")
198
+ if self.debug_mode:
199
+ self.logger.info(f" Insight: {outlier.get('insight', 'N/A')}")
200
+ self.logger.info(f" Impact: {outlier.get('impact', 'N/A')}")
201
+
202
+ # Save all outliers to file in DEBUG mode
203
+ if self.debug_mode:
204
+ outliers_file = self.log_dir / "outliers.json"
205
+ with open(outliers_file, 'w') as f:
206
+ json.dump(self.session_data["outliers"], f, indent=2)
207
 
208
  self.save_session_data()
209
 
 
243
  self.save_session_data()
244
 
245
  def log_stage_completion(self, stage: str, success: bool,
246
+ duration_seconds: float = None, details: str = None):
247
  """Log completion of a demo creation stage"""
248
  stage_data = {
249
  "stage": stage,
250
  "success": success,
251
  "duration": duration_seconds,
252
+ "timestamp": datetime.now().isoformat(),
253
+ "details": details
254
  }
255
 
256
  if "stages" not in self.session_data:
 
259
  self.session_data["stages"].append(stage_data)
260
 
261
  status = "✅ SUCCESS" if success else "❌ FAILED"
262
+ self.logger.info("="*80)
263
+ self.logger.info(f"📍 Stage: {stage.upper()} - {status}")
264
  if duration_seconds:
265
+ self.logger.info(f" Duration: {duration_seconds:.2f} seconds")
266
+ if details:
267
+ self.logger.info(f" Details: {details}")
268
+ self.logger.info("="*80)
269
 
270
  self.save_session_data()
271
 
demo_prep.py CHANGED
@@ -371,20 +371,31 @@ def execute_population_script(python_code, schema_name, skip_modifications=False
371
  ["conn_params.pop('schema', None) # Remove schema to avoid duplicate"]
372
  )
373
  else:
374
- cleaned_code = clean_code
375
- print("✅ Schema pop already in code, skipping injection")
 
 
 
376
 
377
  # Simple and safe schema replacement - just replace the placeholder
378
  cleaned_code = cleaned_code.replace("os.getenv('SNOWFLAKE_SCHEMA')", f"'{schema_name}'")
379
  cleaned_code = cleaned_code.replace('os.getenv("SNOWFLAKE_SCHEMA")', f'"{schema_name}"')
380
 
 
 
 
381
  # FIX: Remove fake.unique() calls that cause "duplicated values after 1,000 iterations" error
382
  cleaned_code = cleaned_code.replace("fake.unique.word()", "fake.word()")
383
  cleaned_code = cleaned_code.replace("fake.unique.email()", "fake.email()")
384
  cleaned_code = cleaned_code.replace("fake.unique.company()", "fake.company()")
385
 
386
  # FIX: Truncate phone numbers to avoid extension overflow (e.g., '790-923-3730x07350')
387
- cleaned_code = cleaned_code.replace("fake.phone_number()", "fake.phone_number()[:20]")
 
 
 
 
 
388
 
389
  # FIX: Convert SQLite-style ? placeholders to Snowflake-style %s placeholders
390
  cleaned_code = re.sub(r'\bVALUES\s*\(\?', 'VALUES (%s', cleaned_code)
@@ -563,6 +574,7 @@ def execute_population_script(python_code, schema_name, skip_modifications=False
563
  "os": os,
564
  "random": random,
565
  "datetime": datetime,
 
566
  "get_snowflake_connection_params": get_snowflake_connection_params,
567
  }
568
 
@@ -574,11 +586,24 @@ def execute_population_script(python_code, schema_name, skip_modifications=False
574
  print(f"🔍 DEBUG: Cleaned code preview (first 500 chars):")
575
  print(cleaned_code[:500])
576
  print("...")
577
-
578
- # Save the full generated code to a file for debugging
579
- with open(f"/tmp/generated_population_script_{schema_name}.py", "w") as f:
 
 
580
  f.write(cleaned_code)
581
- print(f"🔍 DEBUG: Full generated code saved to /tmp/generated_population_script_{schema_name}.py")
 
 
 
 
 
 
 
 
 
 
 
582
 
583
  # First, validate the syntax before executing
584
  try:
@@ -607,6 +632,24 @@ def execute_population_script(python_code, schema_name, skip_modifications=False
607
  print("=" * 50)
608
  print(f"❌ DATA POPULATION FAILED: {str(e)}")
609
  print("=" * 50)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
610
  return False, f"Population execution failed: {str(e)}"
611
 
612
  def generate_deployment_summary(demo_builder_instance):
@@ -1018,7 +1061,7 @@ def create_demo_prep_interface():
1018
  info="Primary AI model for all research and generation tasks"
1019
  )
1020
  default_company_url = gr.Textbox(
1021
- value="https://www.thoughtspot.com",
1022
  label="Default Company URL",
1023
  info="Pre-filled company URL for quick starts"
1024
  )
@@ -1059,6 +1102,12 @@ def create_demo_prep_interface():
1059
  label="Schema Prefix",
1060
  info="Prefix for schema names (e.g., demo_20250930)"
1061
  )
 
 
 
 
 
 
1062
 
1063
  with gr.TabItem("Admin"):
1064
  ts_instance_url = gr.Textbox(
@@ -1262,7 +1311,7 @@ def create_demo_prep_interface():
1262
  # Progressive workflow with Multi-LLM support
1263
  def progressive_workflow_handler(
1264
  url, use_case, llm_provider, demo_builder_instance, mode="expert", stop_before_deploy=False,
1265
- temperature=0.3, max_tokens=4000
1266
  ):
1267
  # Initialize DemoBuilder if none exists
1268
  if demo_builder_instance is None:
@@ -1270,6 +1319,17 @@ def create_demo_prep_interface():
1270
  return "Please enter a company URL", None, "Start Research", False
1271
  demo_builder_instance = DemoBuilder(use_case, url.strip())
1272
 
 
 
 
 
 
 
 
 
 
 
 
1273
  # Initialize or continue with existing content
1274
  if not hasattr(demo_builder_instance, 'accumulated_content'):
1275
  accumulated_content = ""
@@ -1495,6 +1555,10 @@ def create_demo_prep_interface():
1495
  workflow_updates += f"💡 Next: Click 'Create DDL' to generate database schema\n"
1496
  demo_builder_instance.workflow_updates = workflow_updates
1497
 
 
 
 
 
1498
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1499
 
1500
  elif demo_builder_instance.current_stage == "create":
@@ -1506,24 +1570,26 @@ def create_demo_prep_interface():
1506
  )
1507
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1508
 
1509
- # Extract company name for schema naming
1510
- company_name = (
1511
- demo_builder_instance.extract_company_name()
1512
- .replace(" ", "")
1513
- .replace("-", "")
1514
- )
1515
- # Use new DM naming convention for schema
1516
- now = time.localtime()
1517
- yymmdd = time.strftime('%y%m%d', now)
1518
- hhmmss = time.strftime('%H%M%S', now)
1519
-
1520
- # Clean and truncate company name (5 chars)
1521
- company_clean = re.sub(r'[^a-zA-Z0-9]', '', company_name.upper())[:5]
1522
-
1523
- # Clean and truncate use case (3 chars)
1524
- usecase_clean = re.sub(r'[^a-zA-Z0-9]', '', use_case.upper())[:3]
1525
-
1526
- schema_name = f"DM{yymmdd}_{hhmmss}_{company_clean}_{usecase_clean}"
 
 
1527
 
1528
  schema_prompt = f"""Create ONLY complete SQL DDL statements for a {use_case} demo database.
1529
 
@@ -1601,6 +1667,11 @@ Generate complete CREATE TABLE statements with proper Snowflake syntax and depen
1601
  demo_builder_instance.accumulated_content = accumulated_content
1602
  demo_builder_instance.workflow_updates = workflow_updates
1603
 
 
 
 
 
 
1604
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1605
 
1606
  elif demo_builder_instance.current_stage == "populate":
@@ -1830,13 +1901,18 @@ Schema Validation: Will be checked next...
1830
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1831
 
1832
  print(f"🔍 DEBUG: About to advance from populate to deploy stage")
1833
-
1834
  demo_builder_instance.advance_stage()
1835
  demo_builder_instance.accumulated_content = accumulated_content
1836
  demo_builder_instance.workflow_updates = workflow_updates
1837
  print(f"🔍 DEBUG: Advanced to stage: {demo_builder_instance.current_stage}")
1838
  print(f"🔍 DEBUG: Stage status: {demo_builder_instance.stage_status}")
1839
-
 
 
 
 
 
1840
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1841
  return # Exit populate stage immediately after advancing
1842
 
@@ -1876,6 +1952,7 @@ Schema Validation: Will be checked next...
1876
  company_name,
1877
  use_case,
1878
  demo_builder_instance.schema_generation_results,
 
1879
  )
1880
  )
1881
 
@@ -1984,6 +2061,7 @@ Schema Validation: Will be checked next...
1984
  schema=schema_name,
1985
  company_name=company_name,
1986
  use_case=use_case,
 
1987
  progress_callback=ts_detailed_callback
1988
  )
1989
  deployment_complete = True
@@ -2018,25 +2096,9 @@ Schema Validation: Will be checked next...
2018
  demo_builder_instance.deployment_status += ts_progress_msg
2019
  workflow_updates += ts_progress_msg
2020
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
2021
-
2022
- # Add detailed progress messages to main window after deployment
2023
- workflow_updates += "\nBuilding relationships...\n"
2024
- workflow_updates += "Creating tables...\n"
2025
-
2026
- # Show detailed table creation info if available
2027
- if 'tables' in ts_results and ts_results['tables']:
2028
- workflow_updates += f"Generated joins between tables\n"
2029
- for table_name in ts_results['tables']:
2030
- workflow_updates += f" • {table_name} created successfully\n"
2031
- workflow_updates += f"Tables created successfully ({len(ts_results['tables'])} total)\n"
2032
- else:
2033
- workflow_updates += "Generated joins between tables\n"
2034
- workflow_updates += "Tables created successfully\n"
2035
 
2036
- workflow_updates += "Added joins to semantic model\n"
2037
- workflow_updates += "Model created successfully!\n"
2038
- yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
2039
-
2040
  if ts_results['success']:
2041
  ts_msg = "\nThoughtSpot metadata deployment completed!\n\n"
2042
  ts_msg += "Created ThoughtSpot objects:\n"
@@ -2140,9 +2202,19 @@ Schema Validation: Will be checked next...
2140
 
2141
  demo_builder_instance.advance_stage()
2142
  demo_builder_instance.accumulated_content = accumulated_content
 
 
 
 
 
 
 
2143
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
2144
 
2145
  except Exception as e:
 
 
 
2146
  error_msg = f"❌ Deployment error: {str(e)}\n"
2147
  demo_builder_instance.deployment_status += error_msg
2148
  workflow_updates += error_msg
@@ -2158,7 +2230,7 @@ Schema Validation: Will be checked next...
2158
  yield accumulated_content, workflow_updates, None, "Research", False
2159
 
2160
  # Redo functionality
2161
- def redo_latest_stage(demo_builder_instance, llm_provider, url, use_case, mode="expert", stop_deploy=False, temperature=0.3, max_tokens=4000):
2162
  if (
2163
  demo_builder_instance
2164
  and demo_builder_instance.current_stage != "research"
@@ -2173,7 +2245,7 @@ Schema Validation: Will be checked next...
2173
 
2174
  # Re-run the workflow with same format as button_click_handler
2175
  for result in progressive_workflow_handler(
2176
- url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens
2177
  ):
2178
  if len(result) == 5:
2179
  # New format with workflow_updates
@@ -2213,7 +2285,7 @@ Schema Validation: Will be checked next...
2213
 
2214
  url_input = gr.Textbox(
2215
  placeholder="https://www.company.com",
2216
- value="https://www.thoughtspot.com",
2217
  label="Company URL",
2218
  )
2219
 
@@ -2766,7 +2838,7 @@ CREATE TABLE products (
2766
  )
2767
 
2768
  # TESTING HARNESS - Test population code generation without full workflow
2769
- def test_population_generation(mock_schema, use_case, llm_provider="GPT-4"):
2770
  """Generate population code using mock schema for testing"""
2771
  try:
2772
  from main_research import MultiLLMResearcher
@@ -2840,9 +2912,9 @@ SCRIPT REQUIREMENTS:
2840
  # Legacy CDW deployment function removed - deployment now handled by main workflow
2841
 
2842
  # Main click handler with multi-LLM support - UPDATED to route to AI Output tab
2843
- def button_click_handler(url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens):
2844
  for result in progressive_workflow_handler(
2845
- url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens
2846
  ):
2847
  if len(result) == 5:
2848
  # New format with workflow_updates
@@ -2873,7 +2945,7 @@ SCRIPT REQUIREMENTS:
2873
  # Event handlers - UPDATED for new AI Output tab and Results box
2874
  research_btn.click(
2875
  fn=button_click_handler,
2876
- inputs=[url_input, industry_dropdown, llm_dropdown, demo_builder, workflow_mode, stop_before_deploy, temperature_slider, max_tokens_input],
2877
  outputs=[
2878
  ai_output_box, # streaming content goes to AI Output tab
2879
  results_box, # dynamic content based on stage/mode
@@ -2893,10 +2965,10 @@ SCRIPT REQUIREMENTS:
2893
  )
2894
 
2895
  redo_btn.click(
2896
- fn=lambda db, llm, url, use_case, mode, stop_deploy, temp, max_tok: [
2897
- result for result in redo_latest_stage(db, llm, url, use_case, mode, stop_deploy, temp, max_tok)
2898
  ],
2899
- inputs=[demo_builder, llm_dropdown, url_input, industry_dropdown, workflow_mode, stop_before_deploy, temperature_slider, max_tokens_input],
2900
  outputs=[
2901
  ai_output_box, # streaming content goes to AI Output tab
2902
  results_box, # dynamic content based on stage/mode
@@ -3372,7 +3444,7 @@ SCRIPT REQUIREMENTS:
3372
 
3373
  def save_settings_handler(
3374
  email, ai_model, temperature, max_tokens, company_size, company_url, use_case, data_volume,
3375
- warehouse, database, ts_url, ts_username, batch_size, thread_count
3376
  ):
3377
  """Save settings to Supabase database"""
3378
  if not email or '@' not in email:
@@ -3396,7 +3468,8 @@ SCRIPT REQUIREMENTS:
3396
  "thoughtspot_url": ts_url,
3397
  "thoughtspot_username": ts_username,
3398
  "batch_size": batch_size,
3399
- "thread_count": thread_count
 
3400
  }
3401
 
3402
  success = settings_client.save_all_settings(email, settings)
@@ -3424,6 +3497,7 @@ SCRIPT REQUIREMENTS:
3424
  "", # ts_username
3425
  5000, # batch_size
3426
  4, # thread_count
 
3427
  "Settings not loaded - enter valid email", # status
3428
  "gpt-4o", # llm_dropdown (main form AI Model)
3429
  "" # url_input (main form Company URL)
@@ -3447,6 +3521,7 @@ SCRIPT REQUIREMENTS:
3447
  settings.get("thoughtspot_username", ""),
3448
  int(settings.get("batch_size", 5000)),
3449
  int(settings.get("thread_count", 4)),
 
3450
  f"✅ Settings loaded for {email}",
3451
  ai_model, # llm_dropdown (same as default_ai_model)
3452
  company_url # url_input (same as default_company_url)
@@ -3462,6 +3537,7 @@ SCRIPT REQUIREMENTS:
3462
  "",
3463
  5000,
3464
  4,
 
3465
  f"❌ Error loading settings: {str(e)}"
3466
  ]
3467
 
@@ -3488,7 +3564,8 @@ SCRIPT REQUIREMENTS:
3488
  ts_instance_url,
3489
  ts_username,
3490
  batch_size_slider,
3491
- thread_count_slider
 
3492
  ],
3493
  outputs=[settings_status]
3494
  )
@@ -3733,6 +3810,7 @@ SCRIPT REQUIREMENTS:
3733
  ts_username,
3734
  batch_size_slider,
3735
  thread_count_slider,
 
3736
  settings_status,
3737
  llm_dropdown, # Also update main form AI Model dropdown
3738
  url_input # Also update main form Company URL
 
371
  ["conn_params.pop('schema', None) # Remove schema to avoid duplicate"]
372
  )
373
  else:
374
+ # CRITICAL FIX: Add USE SCHEMA command after cursor creation to set schema context
375
+ cleaned_code = cleaned_code.replace(
376
+ "cursor = conn.cursor()",
377
+ f"cursor = conn.cursor()\n cursor.execute('USE SCHEMA \"{schema_name}\"') # Set schema context for INSERTs"
378
+ )
379
 
380
  # Simple and safe schema replacement - just replace the placeholder
381
  cleaned_code = cleaned_code.replace("os.getenv('SNOWFLAKE_SCHEMA')", f"'{schema_name}'")
382
  cleaned_code = cleaned_code.replace('os.getenv("SNOWFLAKE_SCHEMA")', f'"{schema_name}"')
383
 
384
+ # Import re module at the top of this section
385
+ import re
386
+
387
  # FIX: Remove fake.unique() calls that cause "duplicated values after 1,000 iterations" error
388
  cleaned_code = cleaned_code.replace("fake.unique.word()", "fake.word()")
389
  cleaned_code = cleaned_code.replace("fake.unique.email()", "fake.email()")
390
  cleaned_code = cleaned_code.replace("fake.unique.company()", "fake.company()")
391
 
392
  # FIX: Truncate phone numbers to avoid extension overflow (e.g., '790-923-3730x07350')
393
+ # Handle multiple variations of phone number generation - truncate to 15 chars (max field size)
394
+ cleaned_code = cleaned_code.replace("fake.phone_number()", "fake.phone_number()[:15]")
395
+ cleaned_code = re.sub(r"faker\.phone_number\(\)", "faker.phone_number()[:15]", cleaned_code)
396
+ cleaned_code = re.sub(r"'Phone':\s*fake\.phone_number\(\)", "'Phone': fake.phone_number()[:15]", cleaned_code)
397
+ # Also handle when it's in a larger expression or assignment
398
+ cleaned_code = re.sub(r"(\w+\.phone_number\(\))(?!\[)", r"\1[:15]", cleaned_code)
399
 
400
  # FIX: Convert SQLite-style ? placeholders to Snowflake-style %s placeholders
401
  cleaned_code = re.sub(r'\bVALUES\s*\(\?', 'VALUES (%s', cleaned_code)
 
574
  "os": os,
575
  "random": random,
576
  "datetime": datetime,
577
+ "re": re,
578
  "get_snowflake_connection_params": get_snowflake_connection_params,
579
  }
580
 
 
586
  print(f"🔍 DEBUG: Cleaned code preview (first 500 chars):")
587
  print(cleaned_code[:500])
588
  print("...")
589
+
590
+ # ALWAYS save the full generated code for debugging (not just in DEBUG mode)
591
+ # Save to /tmp for immediate access
592
+ tmp_file = f"/tmp/generated_population_script_{schema_name}.py"
593
+ with open(tmp_file, "w") as f:
594
  f.write(cleaned_code)
595
+ print(f"📁 Population script saved to: {tmp_file}")
596
+
597
+ # Also save to logs directory for persistence
598
+ from pathlib import Path
599
+ import datetime
600
+ log_dir = Path("logs") / "population_scripts"
601
+ log_dir.mkdir(parents=True, exist_ok=True)
602
+ timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
603
+ log_file = log_dir / f"population_{schema_name}_{timestamp}.py"
604
+ with open(log_file, "w") as f:
605
+ f.write(cleaned_code)
606
+ print(f"📁 Also saved to: {log_file}")
607
 
608
  # First, validate the syntax before executing
609
  try:
 
632
  print("=" * 50)
633
  print(f"❌ DATA POPULATION FAILED: {str(e)}")
634
  print("=" * 50)
635
+
636
+ # Print detailed traceback
637
+ import traceback
638
+ print("📋 Full traceback:")
639
+ traceback.print_exc()
640
+
641
+ # Show which line of generated code caused the error
642
+ tb = traceback.extract_tb(e.__traceback__)
643
+ for frame in tb:
644
+ if '<population_script>' in frame.filename:
645
+ print(f"\n❌ Error in generated code at line {frame.lineno}:")
646
+ lines = cleaned_code.split('\n')
647
+ start = max(0, frame.lineno - 3)
648
+ end = min(len(lines), frame.lineno + 2)
649
+ for i in range(start, end):
650
+ marker = ">>> " if i == frame.lineno - 1 else " "
651
+ print(f"{marker}{i+1:3d}: {lines[i]}")
652
+
653
  return False, f"Population execution failed: {str(e)}"
654
 
655
  def generate_deployment_summary(demo_builder_instance):
 
1061
  info="Primary AI model for all research and generation tasks"
1062
  )
1063
  default_company_url = gr.Textbox(
1064
+ value="",
1065
  label="Default Company URL",
1066
  info="Pre-filled company URL for quick starts"
1067
  )
 
1102
  label="Schema Prefix",
1103
  info="Prefix for schema names (e.g., demo_20250930)"
1104
  )
1105
+ naming_prefix_input = gr.Textbox(
1106
+ value="",
1107
+ label="Object Naming Prefix (Optional)",
1108
+ placeholder="e.g., BOONE, SALES, DEMO",
1109
+ info="Custom prefix for all objects. Format: {prefix}_{day+ms}_{objtype}. Leave blank for default naming."
1110
+ )
1111
 
1112
  with gr.TabItem("Admin"):
1113
  ts_instance_url = gr.Textbox(
 
1311
  # Progressive workflow with Multi-LLM support
1312
  def progressive_workflow_handler(
1313
  url, use_case, llm_provider, demo_builder_instance, mode="expert", stop_before_deploy=False,
1314
+ temperature=0.3, max_tokens=4000, naming_prefix=""
1315
  ):
1316
  # Initialize DemoBuilder if none exists
1317
  if demo_builder_instance is None:
 
1319
  return "Please enter a company URL", None, "Start Research", False
1320
  demo_builder_instance = DemoBuilder(use_case, url.strip())
1321
 
1322
+ # Initialize demo logger for this session
1323
+ session_logger = get_demo_logger(user_email="demo_user")
1324
+ demo_builder_instance.logger = session_logger
1325
+ session_logger.log_demo_config(
1326
+ use_case=use_case,
1327
+ company=url,
1328
+ company_url=url,
1329
+ industry="TBD",
1330
+ llm_model=llm_provider
1331
+ )
1332
+
1333
  # Initialize or continue with existing content
1334
  if not hasattr(demo_builder_instance, 'accumulated_content'):
1335
  accumulated_content = ""
 
1555
  workflow_updates += f"💡 Next: Click 'Create DDL' to generate database schema\n"
1556
  demo_builder_instance.workflow_updates = workflow_updates
1557
 
1558
+ # Log research stage completion
1559
+ if hasattr(demo_builder_instance, 'logger'):
1560
+ demo_builder_instance.logger.log_stage_completion("research", True)
1561
+
1562
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1563
 
1564
  elif demo_builder_instance.current_stage == "create":
 
1570
  )
1571
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1572
 
1573
+ # Generate schema name using same convention as ThoughtSpot objects
1574
+ from datetime import datetime as dt
1575
+ now = dt.now()
1576
+
1577
+ # Clean prefix (uppercase, alphanumeric only) - can be empty string
1578
+ if naming_prefix:
1579
+ prefix_clean = re.sub(r'[^a-zA-Z0-9]', '', naming_prefix.upper())
1580
+ else:
1581
+ prefix_clean = ""
1582
+
1583
+ # Short unique timestamp: day (2 digits) + microseconds (6 digits)
1584
+ day = now.strftime('%d')
1585
+ microsec = now.strftime('%f') # 6 digits
1586
+ short_timestamp = f"{day}{microsec}"
1587
+
1588
+ # Base format: PREFIX_DDMICROS (prefix can be empty)
1589
+ if prefix_clean:
1590
+ schema_name = f"{prefix_clean}_{short_timestamp}_scm"
1591
+ else:
1592
+ schema_name = f"{short_timestamp}_scm"
1593
 
1594
  schema_prompt = f"""Create ONLY complete SQL DDL statements for a {use_case} demo database.
1595
 
 
1667
  demo_builder_instance.accumulated_content = accumulated_content
1668
  demo_builder_instance.workflow_updates = workflow_updates
1669
 
1670
+ # Log DDL stage completion and save DDL
1671
+ if hasattr(demo_builder_instance, 'logger'):
1672
+ demo_builder_instance.logger.log_ddl(schema_results, demo_builder_instance.schema_name if hasattr(demo_builder_instance, 'schema_name') else None)
1673
+ demo_builder_instance.logger.log_stage_completion("ddl", True)
1674
+
1675
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1676
 
1677
  elif demo_builder_instance.current_stage == "populate":
 
1901
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1902
 
1903
  print(f"🔍 DEBUG: About to advance from populate to deploy stage")
1904
+
1905
  demo_builder_instance.advance_stage()
1906
  demo_builder_instance.accumulated_content = accumulated_content
1907
  demo_builder_instance.workflow_updates = workflow_updates
1908
  print(f"🔍 DEBUG: Advanced to stage: {demo_builder_instance.current_stage}")
1909
  print(f"🔍 DEBUG: Stage status: {demo_builder_instance.stage_status}")
1910
+
1911
+ # Log population code stage completion
1912
+ if hasattr(demo_builder_instance, 'logger') and hasattr(demo_builder_instance, 'population_code'):
1913
+ demo_builder_instance.logger.log_population_code(demo_builder_instance.population_code)
1914
+ demo_builder_instance.logger.log_stage_completion("population_code", True)
1915
+
1916
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
1917
  return # Exit populate stage immediately after advancing
1918
 
 
1952
  company_name,
1953
  use_case,
1954
  demo_builder_instance.schema_generation_results,
1955
+ naming_prefix=naming_prefix
1956
  )
1957
  )
1958
 
 
2061
  schema=schema_name,
2062
  company_name=company_name,
2063
  use_case=use_case,
2064
+ naming_prefix=naming_prefix or None,
2065
  progress_callback=ts_detailed_callback
2066
  )
2067
  deployment_complete = True
 
2096
  demo_builder_instance.deployment_status += ts_progress_msg
2097
  workflow_updates += ts_progress_msg
2098
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2099
 
2100
+ # Deployment progress is shown live during execution - no duplicate summary needed
2101
+
 
 
2102
  if ts_results['success']:
2103
  ts_msg = "\nThoughtSpot metadata deployment completed!\n\n"
2104
  ts_msg += "Created ThoughtSpot objects:\n"
 
2202
 
2203
  demo_builder_instance.advance_stage()
2204
  demo_builder_instance.accumulated_content = accumulated_content
2205
+
2206
+ # Log deployment stage completion
2207
+ if hasattr(demo_builder_instance, 'logger'):
2208
+ deploy_success = deploy_success if 'deploy_success' in locals() else False
2209
+ demo_builder_instance.logger.log_stage_completion("deploy", deploy_success,
2210
+ details=f"Schema: {schema_name if 'schema_name' in locals() else 'N/A'}")
2211
+
2212
  yield accumulated_content, workflow_updates, demo_builder_instance, demo_builder_instance.get_current_button_text(), demo_builder_instance.is_button_disabled()
2213
 
2214
  except Exception as e:
2215
+ # Log deployment error
2216
+ if hasattr(demo_builder_instance, 'logger'):
2217
+ demo_builder_instance.logger.log_error(str(e), "deployment", e)
2218
  error_msg = f"❌ Deployment error: {str(e)}\n"
2219
  demo_builder_instance.deployment_status += error_msg
2220
  workflow_updates += error_msg
 
2230
  yield accumulated_content, workflow_updates, None, "Research", False
2231
 
2232
  # Redo functionality
2233
+ def redo_latest_stage(demo_builder_instance, llm_provider, url, use_case, mode="expert", stop_deploy=False, temperature=0.3, max_tokens=4000, naming_prefix=""):
2234
  if (
2235
  demo_builder_instance
2236
  and demo_builder_instance.current_stage != "research"
 
2245
 
2246
  # Re-run the workflow with same format as button_click_handler
2247
  for result in progressive_workflow_handler(
2248
+ url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens, naming_prefix
2249
  ):
2250
  if len(result) == 5:
2251
  # New format with workflow_updates
 
2285
 
2286
  url_input = gr.Textbox(
2287
  placeholder="https://www.company.com",
2288
+ value="",
2289
  label="Company URL",
2290
  )
2291
 
 
2838
  )
2839
 
2840
  # TESTING HARNESS - Test population code generation without full workflow
2841
+ def test_population_generation(mock_schema, use_case, llm_provider="gpt-4o"):
2842
  """Generate population code using mock schema for testing"""
2843
  try:
2844
  from main_research import MultiLLMResearcher
 
2912
  # Legacy CDW deployment function removed - deployment now handled by main workflow
2913
 
2914
  # Main click handler with multi-LLM support - UPDATED to route to AI Output tab
2915
+ def button_click_handler(url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens, naming_prefix=""):
2916
  for result in progressive_workflow_handler(
2917
+ url, use_case, llm_provider, demo_builder_instance, mode, stop_deploy, temperature, max_tokens, naming_prefix
2918
  ):
2919
  if len(result) == 5:
2920
  # New format with workflow_updates
 
2945
  # Event handlers - UPDATED for new AI Output tab and Results box
2946
  research_btn.click(
2947
  fn=button_click_handler,
2948
+ inputs=[url_input, industry_dropdown, llm_dropdown, demo_builder, workflow_mode, stop_before_deploy, temperature_slider, max_tokens_input, naming_prefix_input],
2949
  outputs=[
2950
  ai_output_box, # streaming content goes to AI Output tab
2951
  results_box, # dynamic content based on stage/mode
 
2965
  )
2966
 
2967
  redo_btn.click(
2968
+ fn=lambda db, llm, url, use_case, mode, stop_deploy, temp, max_tok, naming_prefix: [
2969
+ result for result in redo_latest_stage(db, llm, url, use_case, mode, stop_deploy, temp, max_tok, naming_prefix)
2970
  ],
2971
+ inputs=[demo_builder, llm_dropdown, url_input, industry_dropdown, workflow_mode, stop_before_deploy, temperature_slider, max_tokens_input, naming_prefix_input],
2972
  outputs=[
2973
  ai_output_box, # streaming content goes to AI Output tab
2974
  results_box, # dynamic content based on stage/mode
 
3444
 
3445
  def save_settings_handler(
3446
  email, ai_model, temperature, max_tokens, company_size, company_url, use_case, data_volume,
3447
+ warehouse, database, ts_url, ts_username, batch_size, thread_count, naming_prefix
3448
  ):
3449
  """Save settings to Supabase database"""
3450
  if not email or '@' not in email:
 
3468
  "thoughtspot_url": ts_url,
3469
  "thoughtspot_username": ts_username,
3470
  "batch_size": batch_size,
3471
+ "thread_count": thread_count,
3472
+ "naming_prefix": naming_prefix or ""
3473
  }
3474
 
3475
  success = settings_client.save_all_settings(email, settings)
 
3497
  "", # ts_username
3498
  5000, # batch_size
3499
  4, # thread_count
3500
+ "", # naming_prefix
3501
  "Settings not loaded - enter valid email", # status
3502
  "gpt-4o", # llm_dropdown (main form AI Model)
3503
  "" # url_input (main form Company URL)
 
3521
  settings.get("thoughtspot_username", ""),
3522
  int(settings.get("batch_size", 5000)),
3523
  int(settings.get("thread_count", 4)),
3524
+ settings.get("naming_prefix", ""), # naming_prefix
3525
  f"✅ Settings loaded for {email}",
3526
  ai_model, # llm_dropdown (same as default_ai_model)
3527
  company_url # url_input (same as default_company_url)
 
3537
  "",
3538
  5000,
3539
  4,
3540
+ "", # naming_prefix
3541
  f"❌ Error loading settings: {str(e)}"
3542
  ]
3543
 
 
3564
  ts_instance_url,
3565
  ts_username,
3566
  batch_size_slider,
3567
+ thread_count_slider,
3568
+ naming_prefix_input
3569
  ],
3570
  outputs=[settings_status]
3571
  )
 
3810
  ts_username,
3811
  batch_size_slider,
3812
  thread_count_slider,
3813
+ naming_prefix_input,
3814
  settings_status,
3815
  llm_dropdown, # Also update main form AI Model dropdown
3816
  url_input # Also update main form Company URL
supabase_client.py CHANGED
@@ -355,6 +355,7 @@ def load_gradio_settings(email: str) -> Dict[str, Any]:
355
  "default_data_volume": "Medium (10K rows)",
356
  "default_warehouse": "COMPUTE_WH",
357
  "default_database": "DEMO_DB",
 
358
 
359
  # Data Generation Settings
360
  "fact_table_size": "10000",
 
355
  "default_data_volume": "Medium (10K rows)",
356
  "default_warehouse": "COMPUTE_WH",
357
  "default_database": "DEMO_DB",
358
+ "naming_prefix": "", # Object naming prefix
359
 
360
  # Data Generation Settings
361
  "fact_table_size": "10000",
tests/debug_population_error.py DELETED
@@ -1,245 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Debug script to isolate the 'name e is not defined' error in population code generation.
4
- This will help us understand exactly what's happening without running the full app.
5
- """
6
-
7
- import os
8
- import sys
9
- from dotenv import load_dotenv
10
-
11
- # Add the current directory to Python path
12
- sys.path.append(os.path.dirname(os.path.abspath(__file__)))
13
-
14
- # Import the functions we need to test
15
- try:
16
- from demo_prep import execute_population_script, extract_python_code, validate_python_syntax
17
- from schema_utils import generate_schema_constrained_prompt, parse_ddl_schema
18
- print("✅ Successfully imported required functions")
19
- except ImportError as e:
20
- print(f"❌ Import error: {e}")
21
- sys.exit(1)
22
-
23
- def test_population_error():
24
- """Test the exact scenario that's causing the 'name e is not defined' error"""
25
-
26
- print("🔍 DEBUG TEST: Isolating population script error")
27
- print("=" * 60)
28
-
29
- # Load environment
30
- load_dotenv()
31
-
32
- # Sample DDL that should work (based on previous successful runs)
33
- sample_ddl = """
34
- CREATE TABLE CHANNELS (
35
- CHANNELID INT IDENTITY(1,1) PRIMARY KEY,
36
- NAME VARCHAR(100) NOT NULL,
37
- TYPE VARCHAR(50) NOT NULL,
38
- PERFORMANCEMETRICS VARCHAR(500)
39
- );
40
-
41
- CREATE TABLE CUSTOMERS (
42
- CUSTOMERID INT IDENTITY(1,1) PRIMARY KEY,
43
- NAME VARCHAR(100) NOT NULL,
44
- EMAIL VARCHAR(100) NOT NULL,
45
- PHONE VARCHAR(20),
46
- ADDRESS VARCHAR(200),
47
- PREFERENCES VARCHAR(500),
48
- INTERACTIONHISTORY VARCHAR(1000)
49
- );
50
-
51
- CREATE TABLE PRODUCTS (
52
- PRODUCTID INT IDENTITY(1,1) PRIMARY KEY,
53
- NAME VARCHAR(100) NOT NULL,
54
- CATEGORY VARCHAR(50) NOT NULL,
55
- PRICE DECIMAL(10,2) NOT NULL,
56
- STOCKLEVEL INT NOT NULL
57
- );
58
-
59
- CREATE TABLE SALESTRANSACTIONS (
60
- TRANSACTIONID INT IDENTITY(1,1) PRIMARY KEY,
61
- CUSTOMERID INT NOT NULL,
62
- PRODUCTID INT NOT NULL,
63
- DATE DATE NOT NULL,
64
- AMOUNT DECIMAL(10,2) NOT NULL,
65
- CHANNELID INT NOT NULL,
66
- FOREIGN KEY (CUSTOMERID) REFERENCES CUSTOMERS(CUSTOMERID),
67
- FOREIGN KEY (PRODUCTID) REFERENCES PRODUCTS(PRODUCTID),
68
- FOREIGN KEY (CHANNELID) REFERENCES CHANNELS(CHANNELID)
69
- );
70
-
71
- CREATE TABLE CUSTOMERINTERACTIONS (
72
- INTERACTIONID INT IDENTITY(1,1) PRIMARY KEY,
73
- CUSTOMERID INT NOT NULL,
74
- CHANNELID INT NOT NULL,
75
- DATE DATE NOT NULL,
76
- INTERACTIONTYPE VARCHAR(50) NOT NULL,
77
- OUTCOME VARCHAR(50) NOT NULL,
78
- FOREIGN KEY (CUSTOMERID) REFERENCES CUSTOMERS(CUSTOMERID),
79
- FOREIGN KEY (CHANNELID) REFERENCES CHANNELS(CHANNELID)
80
- );
81
- """
82
-
83
- print("📋 Step 1: Testing DDL parsing...")
84
- try:
85
- schema_info = parse_ddl_schema(sample_ddl)
86
- print(f"✅ DDL parsed successfully: {len(schema_info)} tables found")
87
- for table_name, table_info in schema_info.items():
88
- print(f" - {table_name}: {len(table_info['columns'])} columns")
89
- except Exception as e:
90
- print(f"❌ DDL parsing failed: {e}")
91
- return False
92
-
93
- print("\n📝 Step 2: Testing prompt generation...")
94
- try:
95
- business_context = """
96
- BUSINESS CONTEXT:
97
- - Use Case: Sales AI Analyst
98
- - Target Persona: Sales Operations Manager
99
- - Business Problem: Identifying sales performance gaps and customer churn risks
100
- - Demo Objectives: Demonstrate AI-driven sales insights and predictive analytics
101
- - Success Outcomes: Improved sales forecasting and customer retention
102
-
103
- MANDATORY CONNECTION CODE:
104
- from dotenv import load_dotenv
105
- import os
106
- import snowflake.connector
107
-
108
- load_dotenv()
109
-
110
- from snowflake_auth import get_snowflake_connection_params
111
- conn_params = get_snowflake_connection_params()
112
- conn = snowflake.connector.connect(
113
- **conn_params,
114
- schema=os.getenv('SNOWFLAKE_SCHEMA')
115
- )
116
-
117
- SCRIPT REQUIREMENTS:
118
- 1. Connect to Snowflake using connection code above
119
- 2. Populate tables with realistic data volumes (1000+ rows per table)
120
- 3. Create baseline normal data patterns
121
- 4. NO explanatory text, just executable Python code
122
- 5. Use proper error handling with try-catch blocks
123
- 6. Include data validation to ensure referential integrity
124
- 7. Add progress logging for each table population
125
- 8. Use realistic data patterns that match the business context
126
- 9. Ensure all foreign key relationships are maintained
127
- """
128
-
129
- data_prompt = generate_schema_constrained_prompt(
130
- schema_info,
131
- "Sales AI Analyst",
132
- business_context
133
- )
134
- print(f"✅ Prompt generated successfully: {len(data_prompt)} characters")
135
- except Exception as e:
136
- print(f"❌ Prompt generation failed: {e}")
137
- return False
138
-
139
- print("\n🤖 Step 3: Using sample broken LLM output...")
140
- # Use a sample broken LLM output that likely causes the "name 'e' is not defined" error
141
- llm_output = '''```python
142
- from dotenv import load_dotenv
143
- import os
144
- import snowflake.connector
145
- from snowflake_auth import get_snowflake_connection_params
146
- from faker import Faker
147
- import random
148
-
149
- def main():
150
- try:
151
- load_dotenv()
152
- conn_params = get_snowflake_connection_params()
153
- conn = snowflake.connector.connect(
154
- **conn_params,
155
- schema=os.getenv('SNOWFLAKE_SCHEMA')
156
- )
157
- cursor = conn.cursor()
158
-
159
- # Populate Channels table
160
- print("Populating Channels...")
161
- for i in range(100):
162
- cursor.execute("INSERT INTO CHANNELS (NAME, TYPE) VALUES (?, ?)",
163
- (f"Channel {i}", "Online"))
164
-
165
- conn.commit()
166
- print("Population completed!")
167
-
168
- # This is the broken part - incomplete try-except block
169
- except Exception as e:
170
- print(f"Error: {e}")
171
- # Missing finally or another except block that references 'e'
172
-
173
- if __name__ == "__main__":
174
- main()
175
- ```'''
176
-
177
- print(f"✅ Using sample LLM output: {len(llm_output)} characters")
178
-
179
- # Save the output for inspection
180
- with open("debug_llm_output.txt", "w") as f:
181
- f.write(llm_output)
182
- print("📁 Saved sample LLM output to debug_llm_output.txt")
183
-
184
- print("\n🔍 Step 4: Testing syntax validation...")
185
- try:
186
- syntax_valid, syntax_message = validate_python_syntax(llm_output)
187
- print(f"Syntax validation result: {syntax_valid}")
188
- print(f"Syntax message: {syntax_message}")
189
- except Exception as e:
190
- print(f"❌ Syntax validation failed: {e}")
191
- print(f"❌ This might be our 'name e is not defined' error!")
192
- return False
193
-
194
- print("\n🔧 Step 5: Testing code extraction...")
195
- try:
196
- extracted_code = extract_python_code(llm_output)
197
- print(f"✅ Code extracted: {len(extracted_code)} characters")
198
-
199
- # Save the extracted code for inspection
200
- with open("debug_extracted_code.py", "w") as f:
201
- f.write(extracted_code)
202
- print("📁 Saved extracted code to debug_extracted_code.py")
203
-
204
- except Exception as e:
205
- print(f"❌ Code extraction failed: {e}")
206
- return False
207
-
208
- print("\n🚀 Step 6: Testing population script execution...")
209
- try:
210
- # Use a test schema name
211
- test_schema = "TEST_DEBUG_SCHEMA"
212
-
213
- print(f" Testing with schema: {test_schema}")
214
- success, message = execute_population_script(llm_output, test_schema)
215
-
216
- if success:
217
- print(f"✅ Population script executed successfully!")
218
- else:
219
- print(f"❌ Population script failed: {message}")
220
- print("❌ This is likely where our error occurs!")
221
-
222
- except Exception as e:
223
- print(f"❌ Population script execution failed: {e}")
224
- print(f"❌ Error type: {type(e).__name__}")
225
- import traceback
226
- print("❌ Full traceback:")
227
- traceback.print_exc()
228
- return False
229
-
230
- print("\n" + "=" * 60)
231
- print("🎯 DEBUG TEST COMPLETE")
232
- print("Check the generated files:")
233
- print(" - debug_llm_output.txt (raw LLM response)")
234
- print(" - debug_extracted_code.py (extracted Python code)")
235
- print("=" * 60)
236
-
237
- return True
238
-
239
- if __name__ == "__main__":
240
- print("🧪 Starting isolated debug test for population error...")
241
- success = test_population_error()
242
- if success:
243
- print("✅ Debug test completed successfully!")
244
- else:
245
- print("❌ Debug test failed - this should help identify the issue!")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tests/debug_schema_utils.py DELETED
@@ -1,106 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Debug script to isolate the 'name e is not defined' error in schema_utils.py
4
- """
5
-
6
- import os
7
- import sys
8
- from dotenv import load_dotenv
9
-
10
- # Add the current directory to Python path
11
- sys.path.append(os.path.dirname(os.path.abspath(__file__)))
12
-
13
- # Import the functions we need to test
14
- try:
15
- from schema_utils import generate_schema_constrained_prompt, parse_ddl_schema
16
- print("✅ Successfully imported schema_utils functions")
17
- except ImportError as e:
18
- print(f"❌ Import error: {e}")
19
- sys.exit(1)
20
-
21
- def test_schema_utils_error():
22
- """Test the generate_schema_constrained_prompt function specifically"""
23
-
24
- print("🔍 DEBUG TEST: Isolating schema_utils error")
25
- print("=" * 60)
26
-
27
- # Load environment
28
- load_dotenv()
29
-
30
- # Sample DDL
31
- sample_ddl = """
32
- CREATE TABLE CHANNELS (
33
- CHANNELID INT IDENTITY(1,1) PRIMARY KEY,
34
- NAME VARCHAR(100) NOT NULL,
35
- TYPE VARCHAR(50) NOT NULL,
36
- PERFORMANCEMETRICS VARCHAR(500)
37
- );
38
-
39
- CREATE TABLE CUSTOMERS (
40
- CUSTOMERID INT IDENTITY(1,1) PRIMARY KEY,
41
- NAME VARCHAR(100) NOT NULL,
42
- EMAIL VARCHAR(100) NOT NULL,
43
- PHONE VARCHAR(20),
44
- ADDRESS VARCHAR(200),
45
- PREFERENCES VARCHAR(500),
46
- INTERACTIONHISTORY VARCHAR(1000)
47
- );
48
- """
49
-
50
- print("📋 Step 1: Testing DDL parsing...")
51
- try:
52
- schema_info = parse_ddl_schema(sample_ddl)
53
- print(f"✅ DDL parsed successfully: {len(schema_info)} tables found")
54
- for table_name, table_info in schema_info.items():
55
- print(f" - {table_name}: {len(table_info['columns'])} columns")
56
- print(f" Columns: {[col['name'] for col in table_info['columns']]}")
57
- except Exception as e:
58
- print(f"❌ DDL parsing failed: {e}")
59
- import traceback
60
- print("❌ Full traceback:")
61
- traceback.print_exc()
62
- return False
63
-
64
- print("\n📝 Step 2: Testing prompt generation...")
65
- try:
66
- business_context = """
67
- BUSINESS CONTEXT:
68
- - Use Case: Sales AI Analyst
69
- - Target Persona: Sales Operations Manager
70
- - Business Problem: Identifying sales performance gaps
71
- """
72
-
73
- print(" Calling generate_schema_constrained_prompt...")
74
- data_prompt = generate_schema_constrained_prompt(
75
- schema_info,
76
- "Sales AI Analyst",
77
- business_context
78
- )
79
- print(f"✅ Prompt generated successfully: {len(data_prompt)} characters")
80
-
81
- # Save the prompt for inspection
82
- with open("debug_generated_prompt.txt", "w") as f:
83
- f.write(data_prompt)
84
- print("📁 Saved generated prompt to debug_generated_prompt.txt")
85
-
86
- except Exception as e:
87
- print(f"❌ Prompt generation failed: {e}")
88
- print(f"❌ Error type: {type(e).__name__}")
89
- import traceback
90
- print("❌ Full traceback:")
91
- traceback.print_exc()
92
- return False
93
-
94
- print("\n" + "=" * 60)
95
- print("🎯 SCHEMA_UTILS DEBUG TEST COMPLETE")
96
- print("=" * 60)
97
-
98
- return True
99
-
100
- if __name__ == "__main__":
101
- print("🧪 Starting isolated debug test for schema_utils error...")
102
- success = test_schema_utils_error()
103
- if success:
104
- print("✅ Debug test completed successfully!")
105
- else:
106
- print("❌ Debug test failed - this should help identify the issue!")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tests/test_connection_success.py DELETED
@@ -1,123 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- SUCCESS-ONLY ThoughtSpot Connection Test
4
- GUARANTEED SUCCESS - Connection creation only
5
- """
6
-
7
- import os
8
- import sys
9
- # Add parent directory to path
10
- sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
11
-
12
- from dotenv import load_dotenv
13
- from thoughtspot_deployer import ThoughtSpotDeployer
14
-
15
- load_dotenv()
16
-
17
- def test_connection_success():
18
- """Test ONLY connection creation - GUARANTEED SUCCESS"""
19
-
20
- print("🎯 THOUGHTSPOT CONNECTION SUCCESS TEST")
21
- print("=" * 55)
22
- print("🎉 This test focuses ONLY on what works perfectly!")
23
-
24
- schema_name = "20250918_001404_THOUG_SAL"
25
- database = "DEMOBUILD"
26
-
27
- print(f"📊 Testing with schema: {schema_name}")
28
- print(f"📊 Database: {database}")
29
- print()
30
-
31
- try:
32
- # Initialize deployer
33
- deployer = ThoughtSpotDeployer()
34
- print("✅ ThoughtSpot deployer initialized")
35
-
36
- # Test authentication
37
- print("\n🔐 Testing ThoughtSpot authentication...")
38
- if deployer.authenticate():
39
- print("✅ ThoughtSpot authentication SUCCESSFUL")
40
- else:
41
- print("❌ ThoughtSpot authentication failed")
42
- return False
43
-
44
- # Test connection creation ONLY
45
- print("\n🔗 Testing connection creation...")
46
-
47
- # Create connection using the internal method directly
48
- from datetime import datetime
49
- connection_name = f"success_conn_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
50
-
51
- print(f"🔗 Creating connection: {connection_name}")
52
- print(f" Account: '{deployer.sf_account}'")
53
- print(f" User: '{deployer.sf_user}'")
54
- print(f" Database: '{database}'")
55
-
56
- # Create the connection TML
57
- connection_tml_yaml = deployer.create_connection_tml(connection_name, database)
58
-
59
- # Deploy the connection
60
- response = deployer.session.post(
61
- f"{deployer.base_url}/api/rest/2.0/metadata/tml/import",
62
- json={
63
- "metadata_tmls": [connection_tml_yaml],
64
- "import_policy": "PARTIAL"
65
- }
66
- )
67
-
68
- print(f" Response status: {response.status_code}")
69
-
70
- if response.status_code == 200:
71
- result = response.json()
72
- if isinstance(result, list) and len(result) > 0:
73
- if result[0].get('response', {}).get('status', {}).get('status_code') == 'OK':
74
- connection_guid = result[0].get('response', {}).get('header', {}).get('id_guid')
75
-
76
- print("🎉" * 20)
77
- print("🎯 CONNECTION CREATION SUCCESS!")
78
- print(f"✅ Connection Name: {connection_name}")
79
- print(f"✅ Connection GUID: {connection_guid}")
80
- print(f"✅ Schema Access: {schema_name}")
81
- print(f"✅ Authentication: KEY_PAIR working")
82
- print(f"✅ API Integration: PERFECT")
83
- print("🎉" * 20)
84
-
85
- return True
86
- else:
87
- error_msg = result[0].get('response', {}).get('status', {}).get('error_message', 'Unknown error')
88
- print(f"❌ Connection creation failed: {error_msg}")
89
- return False
90
- else:
91
- print(f"❌ Unexpected response format: {result}")
92
- return False
93
- else:
94
- try:
95
- error_response = response.json()
96
- print(f"❌ HTTP {response.status_code}: {error_response}")
97
- except:
98
- print(f"❌ HTTP {response.status_code}: {response.text}")
99
- return False
100
-
101
- except Exception as e:
102
- print(f"❌ Test failed with exception: {str(e)}")
103
- import traceback
104
- traceback.print_exc()
105
- return False
106
-
107
- def main():
108
- """Run the success test"""
109
- success = test_connection_success()
110
-
111
- print("\n" + "="*55)
112
- if success:
113
- print("🏆 OVERALL RESULT: COMPLETE SUCCESS! 🏆")
114
- print("🚀 ThoughtSpot integration is working perfectly!")
115
- print("🎯 Ready for production deployment!")
116
- else:
117
- print("❌ OVERALL RESULT: Something failed")
118
- print("="*55)
119
-
120
- return success
121
-
122
- if __name__ == "__main__":
123
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tests/test_core_success.py DELETED
@@ -1,150 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- CORE SUCCESS ThoughtSpot Test
4
- Skip problematic TIMESTAMP columns, focus on core columns
5
- Connection + Tables + Model = SUCCESS
6
- """
7
-
8
- import os
9
- import sys
10
- # Add parent directory to path
11
- sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
12
-
13
- from dotenv import load_dotenv
14
- from thoughtspot_deployer import ThoughtSpotDeployer
15
-
16
- load_dotenv()
17
-
18
- def test_core_success():
19
- """Test with core columns only (skip CREATEDAT timestamps)"""
20
-
21
- print("🎯 THOUGHTSPOT CORE SUCCESS TEST")
22
- print("=" * 55)
23
- print("🎉 Using CORE columns only (skip CREATEDAT)!")
24
-
25
- schema_name = "20250918_001404_THOUG_SAL"
26
- database = "DEMOBUILD"
27
-
28
- # Core DDL - skip CREATEDAT columns that cause timestamp issues
29
- core_ddl = """
30
- CREATE TABLE CUSTOMERS (
31
- CUSTOMERID NUMBER(38,0) NOT NULL,
32
- FIRSTNAME VARCHAR(50) NOT NULL,
33
- LASTNAME VARCHAR(50) NOT NULL,
34
- EMAIL VARCHAR(100) NOT NULL,
35
- PHONE VARCHAR(20) NOT NULL,
36
- ADDRESS VARCHAR(255) NOT NULL,
37
- CITY VARCHAR(50) NOT NULL,
38
- STATE VARCHAR(50) NOT NULL,
39
- ZIPCODE VARCHAR(10) NOT NULL,
40
- COUNTRY VARCHAR(50) NOT NULL
41
- );
42
-
43
- CREATE TABLE PRODUCTS (
44
- PRODUCTID NUMBER(38,0) NOT NULL,
45
- PRODUCTNAME VARCHAR(100) NOT NULL,
46
- DESCRIPTION VARCHAR(255) NOT NULL,
47
- PRICE NUMBER(10,2) NOT NULL,
48
- STOCKQUANTITY NUMBER(38,0) NOT NULL
49
- );
50
-
51
- CREATE TABLE ORDERS (
52
- ORDERID NUMBER(38,0) NOT NULL,
53
- CUSTOMERID NUMBER(38,0) NOT NULL,
54
- ORDERDATE DATE NOT NULL,
55
- TOTALAMOUNT NUMBER(10,2) NOT NULL,
56
- STATUS VARCHAR(50) NOT NULL
57
- );
58
- """
59
-
60
- print(f"📊 Testing with schema: {schema_name}")
61
- print(f"📊 Database: {database}")
62
- print(f"📊 Tables: 3 core tables (no CREATEDAT columns)")
63
- print()
64
-
65
- try:
66
- # Initialize deployer
67
- deployer = ThoughtSpotDeployer()
68
- print("✅ ThoughtSpot deployer initialized")
69
-
70
- # Test authentication
71
- print("\n🔐 Testing ThoughtSpot authentication...")
72
- if deployer.authenticate():
73
- print("✅ ThoughtSpot authentication SUCCESSFUL")
74
- else:
75
- print("❌ ThoughtSpot authentication failed")
76
- return False
77
-
78
- # Test FULL deployment
79
- print("\n🚀 Testing CORE deployment (Connection + Tables + Model)...")
80
-
81
- results = deployer.deploy_all(
82
- ddl=core_ddl,
83
- database=database,
84
- schema=schema_name
85
- )
86
-
87
- print(f"\n📋 RESULTS:")
88
-
89
- # Connection
90
- if results.get('connection'):
91
- print(f"✅ Connection: {results['connection']}")
92
- else:
93
- print("❌ Connection: Failed")
94
-
95
- # Tables
96
- if results.get('tables'):
97
- print(f"✅ Tables: {len(results['tables'])} created")
98
- for table in results['tables']:
99
- print(f" 📊 {table}")
100
- else:
101
- print("❌ Tables: None created")
102
- if results.get('errors'):
103
- print(" 🔍 First few errors:")
104
- for error in results['errors'][:2]:
105
- print(f" • {error[:80]}...")
106
-
107
- # Model
108
- if results.get('model'):
109
- print(f"✅ Model: {results['model']}")
110
- else:
111
- print("❌ Model: None created")
112
-
113
- # Success evaluation
114
- connection_success = bool(results.get('connection'))
115
- tables_success = len(results.get('tables', [])) > 0
116
- model_success = bool(results.get('model'))
117
-
118
- print(f"\n🎯 SUCCESS BREAKDOWN:")
119
- print(f" Connection: {'✅' if connection_success else '❌'}")
120
- print(f" Tables: {'✅' if tables_success else '❌'}")
121
- print(f" Model: {'✅' if model_success else '❌'}")
122
-
123
- overall_success = connection_success and tables_success
124
-
125
- print("\n" + "="*55)
126
- if overall_success:
127
- print("🏆 OVERALL RESULT: SUCCESS! 🏆")
128
- print("🎉 Connection + Tables working!")
129
- if model_success:
130
- print("🎯 COMPLETE SUCCESS - Model too!")
131
- else:
132
- print("🔧 Model needs relationships, but CORE SUCCESS!")
133
- else:
134
- print("❌ OVERALL RESULT: Still working on it...")
135
- print("="*55)
136
-
137
- return overall_success
138
-
139
- except Exception as e:
140
- print(f"❌ Test failed with exception: {str(e)}")
141
- import traceback
142
- traceback.print_exc()
143
- return False
144
-
145
- def main():
146
- """Run the core success test"""
147
- return test_core_success()
148
-
149
- if __name__ == "__main__":
150
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tests/test_deployment_flow.py DELETED
@@ -1,54 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Test the updated deployment flow with schema permissions
4
- """
5
-
6
- import os
7
- from dotenv import load_dotenv
8
-
9
- # Load environment variables (you may need to copy .env from sandbox2026)
10
- load_dotenv()
11
-
12
- def test_deployment_flow():
13
- """Test the complete deployment flow"""
14
-
15
- print("🧪 Testing ThoughtSpot Deployment Flow")
16
- print("=" * 50)
17
-
18
- # Check environment variables
19
- required_vars = [
20
- 'THOUGHTSPOT_URL', 'THOUGHTSPOT_USERNAME', 'THOUGHTSPOT_PASSWORD',
21
- 'SNOWFLAKE_ACCOUNT', 'SNOWFLAKE_USER', 'SNOWFLAKE_PASSWORD',
22
- 'SNOWFLAKE_ROLE', 'SNOWFLAKE_WAREHOUSE'
23
- ]
24
-
25
- missing_vars = []
26
- for var in required_vars:
27
- if not os.getenv(var):
28
- missing_vars.append(var)
29
-
30
- if missing_vars:
31
- print("❌ Missing environment variables:")
32
- for var in missing_vars:
33
- print(f" - {var}")
34
- print("\n💡 Copy your .env file from sandbox2026 or set these variables")
35
- return False
36
-
37
- print("✅ All environment variables found")
38
-
39
- # Show the intended flow
40
- print("\n🚀 Deployment Flow:")
41
- print("0️⃣ Grant schema permissions to se_role")
42
- print(" GRANT USAGE ON DATABASE DEMOBUILD TO ROLE se_role")
43
- print(" GRANT USAGE ON SCHEMA DEMOBUILD.{schema} TO ROLE se_role")
44
- print(" GRANT SELECT ON ALL TABLES IN SCHEMA DEMOBUILD.{schema} TO ROLE se_role")
45
- print()
46
- print("1️⃣ Create ThoughtSpot connection")
47
- print("2️⃣ Create tables from DDL")
48
- print("3️⃣ Create model")
49
-
50
- print("\n✅ Flow validated - ready for deployment!")
51
- return True
52
-
53
- if __name__ == "__main__":
54
- test_deployment_flow()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tests/test_end_to_end_demo.py DELETED
@@ -1,174 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Test End-to-End Demo System
4
- Complete deployment test for creating a full demo model
5
- """
6
-
7
- import os
8
- import sys
9
- sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
10
-
11
- from datetime import datetime
12
- from dotenv import load_dotenv
13
- from thoughtspot_deployer import ThoughtSpotDeployer
14
-
15
- load_dotenv()
16
-
17
- def test_end_to_end_demo():
18
- """Test complete end-to-end demo deployment"""
19
-
20
- print("🚀 END-TO-END DEMO SYSTEM DEPLOYMENT")
21
- print("=" * 60)
22
- print("🎯 Creating complete demo model with all tables and proper structure")
23
-
24
- # Use the working schema
25
- schema_name = "20250918_161144_THOUG_SAL"
26
- database = "DEMOBUILD"
27
-
28
- # Complete DDL for full e-commerce demo
29
- demo_ddl = """
30
- CREATE TABLE CUSTOMERS (
31
- CUSTOMERID NUMBER(38,0) NOT NULL,
32
- FIRSTNAME VARCHAR(100) NOT NULL,
33
- LASTNAME VARCHAR(100) NOT NULL,
34
- EMAIL VARCHAR(255) NOT NULL,
35
- PHONE VARCHAR(20) NOT NULL,
36
- REGISTRATIONDATE DATE NOT NULL
37
- );
38
-
39
- CREATE TABLE PRODUCTS (
40
- PRODUCTID NUMBER(38,0) NOT NULL,
41
- PRODUCTNAME VARCHAR(255) NOT NULL,
42
- CATEGORY VARCHAR(100) NOT NULL,
43
- PRICE NUMBER(10,2) NOT NULL,
44
- STOCKQUANTITY NUMBER(38,0) NOT NULL
45
- );
46
-
47
- CREATE TABLE ORDERS (
48
- ORDERID NUMBER(38,0) NOT NULL,
49
- CUSTOMERID NUMBER(38,0) NOT NULL,
50
- ORDERDATE TIMESTAMP_NTZ(9) NOT NULL,
51
- TOTALAMOUNT NUMBER(10,2) NOT NULL
52
- );
53
-
54
- CREATE TABLE ORDERITEMS (
55
- ORDERITEMID NUMBER(38,0) NOT NULL,
56
- ORDERID NUMBER(38,0) NOT NULL,
57
- PRODUCTID NUMBER(38,0) NOT NULL,
58
- QUANTITY NUMBER(38,0) NOT NULL,
59
- UNITPRICE NUMBER(10,2) NOT NULL
60
- );
61
-
62
- CREATE TABLE SALES (
63
- SALEID NUMBER(38,0) NOT NULL,
64
- ORDERID NUMBER(38,0) NOT NULL,
65
- SALESREPID NUMBER(38,0) NOT NULL,
66
- SALEDATE TIMESTAMP_NTZ(9) NOT NULL
67
- );
68
-
69
- CREATE TABLE SALESREPS (
70
- SALESREPID NUMBER(38,0) NOT NULL,
71
- FIRSTNAME VARCHAR(100) NOT NULL,
72
- LASTNAME VARCHAR(100) NOT NULL,
73
- EMAIL VARCHAR(255) NOT NULL,
74
- PHONE VARCHAR(20) NOT NULL,
75
- HIREDATE DATE NOT NULL
76
- );
77
- """
78
-
79
- print(f"📊 Deploying to schema: {schema_name}")
80
- print(f"📊 Database: {database}")
81
- print(f"📊 Tables: 6 tables (complete e-commerce model)")
82
- print()
83
-
84
- try:
85
- # Initialize deployer
86
- deployer = ThoughtSpotDeployer()
87
-
88
- # Create unique names for this demo
89
- timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
90
- connection_name = f"demo_system_{timestamp}"
91
-
92
- print(f"🔗 Demo Connection: {connection_name}")
93
- print(f"🏷️ This will be your complete demo system!")
94
- print()
95
-
96
- # Deploy everything using the updated deployer
97
- print("🚀 Starting end-to-end deployment...")
98
- results = deployer.deploy_all(
99
- ddl=demo_ddl,
100
- database=database,
101
- schema=schema_name,
102
- connection_name=connection_name
103
- )
104
-
105
- print("\n" + "="*70)
106
- print("📋 END-TO-END DEPLOYMENT RESULTS")
107
- print("="*70)
108
-
109
- if results['success']:
110
- print("🎉 DEPLOYMENT SUCCESSFUL!")
111
- print(f"✅ Connection: {results['connection']}")
112
- print(f"✅ Tables created: {len(results['tables'])}")
113
- for table in results['tables']:
114
- print(f" • {table}")
115
- print(f"✅ Model: {results['model']}")
116
-
117
- print("\n🎯 YOUR DEMO SYSTEM IS READY!")
118
- print("="*50)
119
- print(f"🔗 Connection Name: '{results['connection']}'")
120
- print(f"📊 Model Name: '{results['model']}'")
121
- print()
122
- print("✨ WHAT YOU CAN DO NOW:")
123
- print("1. 📈 Create searches and answers using the model")
124
- print("2. 🎨 Build dashboards with the data")
125
- print("3. 🤖 Use the model for AI-powered analytics")
126
- print("4. 👥 Share with demo users")
127
- print()
128
- print("🔍 MODEL FEATURES:")
129
- print("• ✅ No ID columns cluttering the interface")
130
- print("• ✅ Smart column naming (RepFirstname, repPhone, etc.)")
131
- print("• ✅ Proper measures vs attributes")
132
- print("• ✅ Clean, business-friendly structure")
133
- print("• ✅ Matches your proven boone_test5 pattern")
134
-
135
- else:
136
- print("❌ DEPLOYMENT FAILED")
137
- if results['errors']:
138
- print("Errors encountered:")
139
- for i, error in enumerate(results['errors'], 1):
140
- print(f" {i}. {error}")
141
-
142
- print("\n🔧 TROUBLESHOOTING:")
143
- print("• Check ThoughtSpot connection settings")
144
- print("• Verify Snowflake schema exists and is accessible")
145
- print("• Ensure proper permissions are set")
146
-
147
- return results['success']
148
-
149
- except Exception as e:
150
- print(f"\n❌ Deployment failed with error: {e}")
151
- import traceback
152
- traceback.print_exc()
153
-
154
- print("\n🔧 COMMON ISSUES:")
155
- print("• Authentication failure - check credentials")
156
- print("• Schema not found - run demo_prep.py first")
157
- print("• Permission issues - verify Snowflake roles")
158
-
159
- return False
160
-
161
- if __name__ == "__main__":
162
- print("🎬 STARTING END-TO-END DEMO SYSTEM DEPLOYMENT")
163
- print("This will create a complete ThoughtSpot demo environment!")
164
- print()
165
-
166
- success = test_end_to_end_demo()
167
-
168
- if success:
169
- print(f"\n🎉 SUCCESS! Your end-to-end demo system is ready to use!")
170
- print("Check ThoughtSpot for your new connection and model.")
171
- else:
172
- print(f"\n❌ FAILED! Please check the error messages above.")
173
-
174
- sys.exit(0 if success else 1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tests/test_framework.py DELETED
@@ -1,447 +0,0 @@
1
- #!/usr/bin/env python3
2
- """
3
- Enhanced Test Framework for Demo Wire
4
- Comprehensive testing with schema validation, performance metrics, and robust reporting
5
- """
6
-
7
- import os
8
- import sys
9
- import time
10
- import json
11
- import traceback
12
- from datetime import datetime
13
- from typing import Dict, List, Any, Optional, Tuple
14
- from dataclasses import dataclass, asdict
15
- from enum import Enum
16
-
17
- # Add parent directory to path
18
- sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
19
-
20
- from dotenv import load_dotenv
21
- load_dotenv()
22
-
23
- class TestResult(Enum):
24
- PASS = "✅ PASS"
25
- FAIL = "❌ FAIL"
26
- SKIP = "⏭️ SKIP"
27
- WARN = "⚠️ WARN"
28
-
29
- @dataclass
30
- class TestCase:
31
- """Individual test case with metadata"""
32
- name: str
33
- description: str
34
- category: str
35
- priority: int # 1=critical, 2=important, 3=nice-to-have
36
- timeout: int = 60 # seconds
37
- prerequisites: List[str] = None
38
- expected_duration: float = 0.0 # seconds
39
-
40
- def __post_init__(self):
41
- if self.prerequisites is None:
42
- self.prerequisites = []
43
-
44
- @dataclass
45
- class TestReport:
46
- """Test execution report"""
47
- test_name: str
48
- result: TestResult
49
- duration: float
50
- error_message: str = ""
51
- details: Dict[str, Any] = None
52
- timestamp: str = ""
53
-
54
- def __post_init__(self):
55
- if self.details is None:
56
- self.details = {}
57
- if not self.timestamp:
58
- self.timestamp = datetime.now().isoformat()
59
-
60
- class TestFramework:
61
- """Enhanced test framework with comprehensive reporting and validation"""
62
-
63
- def __init__(self):
64
- self.results: List[TestReport] = []
65
- self.start_time = None
66
- self.test_cases: List[TestCase] = []
67
- self.setup_complete = False
68
-
69
- def add_test(self, test_case: TestCase):
70
- """Register a test case"""
71
- self.test_cases.append(test_case)
72
-
73
- def validate_environment(self) -> TestReport:
74
- """Validate required environment variables and dependencies"""
75
- start_time = time.time()
76
-
77
- required_vars = [
78
- 'THOUGHTSPOT_URL', 'THOUGHTSPOT_USERNAME', 'THOUGHTSPOT_PASSWORD',
79
- 'SNOWFLAKE_ACCOUNT', 'SNOWFLAKE_USER', 'SNOWFLAKE_PRIVATE_KEY',
80
- 'SNOWFLAKE_ROLE', 'SNOWFLAKE_WAREHOUSE'
81
- ]
82
-
83
- missing_vars = []
84
- for var in required_vars:
85
- if not os.getenv(var):
86
- missing_vars.append(var)
87
-
88
- duration = time.time() - start_time
89
-
90
- if missing_vars:
91
- return TestReport(
92
- test_name="Environment Validation",
93
- result=TestResult.FAIL,
94
- duration=duration,
95
- error_message=f"Missing environment variables: {', '.join(missing_vars)}",
96
- details={"missing_vars": missing_vars}
97
- )
98
-
99
- return TestReport(
100
- test_name="Environment Validation",
101
- result=TestResult.PASS,
102
- duration=duration,
103
- details={"validated_vars": len(required_vars)}
104
- )
105
-
106
- def validate_schema_structure(self, ddl: str) -> TestReport:
107
- """Validate DDL structure and syntax"""
108
- start_time = time.time()
109
-
110
- try:
111
- # Basic DDL validation
112
- validation_results = {
113
- "has_create_statements": "CREATE TABLE" in ddl.upper(),
114
- "table_count": ddl.upper().count("CREATE TABLE"),
115
- "has_primary_keys": "PRIMARY KEY" in ddl.upper(),
116
- "has_not_null": "NOT NULL" in ddl.upper(),
117
- "estimated_complexity": len(ddl.split('\n'))
118
- }
119
-
120
- # Check for common DDL patterns
121
- critical_issues = []
122
- warnings = []
123
-
124
- if validation_results["table_count"] == 0:
125
- critical_issues.append("No CREATE TABLE statements found")
126
-
127
- if not validation_results["has_not_null"]:
128
- warnings.append("No NOT NULL constraints found - consider data quality")
129
-
130
- # Validate column naming patterns
131
- lines = ddl.split('\n')
132
- for line in lines:
133
- if 'CREATE TABLE' in line.upper():
134
- table_name = line.split()[-2] if '(' in line else line.split()[-1]
135
- validation_results[f"table_{table_name}"] = {
136
- "found": True,
137
- "line": line.strip()
138
- }
139
-
140
- duration = time.time() - start_time
141
- result = TestResult.FAIL if critical_issues else (TestResult.WARN if warnings else TestResult.PASS)
142
-
143
- return TestReport(
144
- test_name="Schema Structure Validation",
145
- result=result,
146
- duration=duration,
147
- error_message="; ".join(critical_issues) if critical_issues else "",
148
- details={
149
- **validation_results,
150
- "warnings": warnings,
151
- "critical_issues": critical_issues
152
- }
153
- )
154
-
155
- except Exception as e:
156
- return TestReport(
157
- test_name="Schema Structure Validation",
158
- result=TestResult.FAIL,
159
- duration=time.time() - start_time,
160
- error_message=f"Validation failed: {str(e)}"
161
- )
162
-
163
- def test_thoughtspot_auth(self) -> TestReport:
164
- """Test ThoughtSpot authentication"""
165
- start_time = time.time()
166
-
167
- try:
168
- from thoughtspot_deployer import ThoughtSpotDeployer
169
-
170
- deployer = ThoughtSpotDeployer()
171
- auth_success = deployer.authenticate()
172
-
173
- duration = time.time() - start_time
174
-
175
- return TestReport(
176
- test_name="ThoughtSpot Authentication",
177
- result=TestResult.PASS if auth_success else TestResult.FAIL,
178
- duration=duration,
179
- error_message="" if auth_success else "Authentication failed",
180
- details={
181
- "auth_method": "session_based",
182
- "response_time": duration
183
- }
184
- )
185
-
186
- except Exception as e:
187
- return TestReport(
188
- test_name="ThoughtSpot Authentication",
189
- result=TestResult.FAIL,
190
- duration=time.time() - start_time,
191
- error_message=f"Auth test failed: {str(e)}"
192
- )
193
-
194
- def test_snowflake_connection(self) -> TestReport:
195
- """Test Snowflake key pair authentication"""
196
- start_time = time.time()
197
-
198
- try:
199
- import snowflake.connector
200
- from snowflake_auth import get_snowflake_connection_params
201
-
202
- params = get_snowflake_connection_params()
203
- conn = snowflake.connector.connect(**params)
204
- cursor = conn.cursor()
205
-
206
- # Test basic query
207
- cursor.execute("SELECT CURRENT_VERSION()")
208
- version = cursor.fetchone()[0]
209
-
210
- conn.close()
211
- duration = time.time() - start_time
212
-
213
- return TestReport(
214
- test_name="Snowflake Connection",
215
- result=TestResult.PASS,
216
- duration=duration,
217
- details={
218
- "snowflake_version": version,
219
- "auth_method": "key_pair",
220
- "connection_time": duration
221
- }
222
- )
223
-
224
- except Exception as e:
225
- return TestReport(
226
- test_name="Snowflake Connection",
227
- result=TestResult.FAIL,
228
- duration=time.time() - start_time,
229
- error_message=f"Snowflake connection failed: {str(e)}"
230
- )
231
-
232
- def test_deployment_pipeline(self, ddl: str, database: str, schema: str) -> TestReport:
233
- """Test complete deployment pipeline"""
234
- start_time = time.time()
235
-
236
- try:
237
- from thoughtspot_deployer import ThoughtSpotDeployer
238
-
239
- deployer = ThoughtSpotDeployer()
240
-
241
- # Authenticate first
242
- if not deployer.authenticate():
243
- return TestReport(
244
- test_name="Deployment Pipeline",
245
- result=TestResult.FAIL,
246
- duration=time.time() - start_time,
247
- error_message="Authentication failed"
248
- )
249
-
250
- # Run deployment
251
- results = deployer.deploy_all(
252
- ddl=ddl,
253
- database=database,
254
- schema=schema
255
- )
256
-
257
- duration = time.time() - start_time
258
-
259
- # Analyze results
260
- pipeline_success = {
261
- "connection": bool(results.get('connection')),
262
- "tables": len(results.get('tables', [])) > 0,
263
- "worksheet": bool(results.get('worksheet')),
264
- "errors": results.get('errors', [])
265
- }
266
-
267
- overall_success = pipeline_success["connection"] and pipeline_success["tables"]
268
-
269
- return TestReport(
270
- test_name="Deployment Pipeline",
271
- result=TestResult.PASS if overall_success else TestResult.FAIL,
272
- duration=duration,
273
- error_message=f"Errors: {len(pipeline_success['errors'])}" if pipeline_success['errors'] else "",
274
- details={
275
- **pipeline_success,
276
- "deployment_time": duration,
277
- "schema_used": schema,
278
- "tables_created": len(results.get('tables', []))
279
- }
280
- )
281
-
282
- except Exception as e:
283
- return TestReport(
284
- test_name="Deployment Pipeline",
285
- result=TestResult.FAIL,
286
- duration=time.time() - start_time,
287
- error_message=f"Pipeline test failed: {str(e)}"
288
- )
289
-
290
- def run_performance_test(self, test_func, *args, **kwargs) -> Tuple[Any, float, bool]:
291
- """Run a test function with performance monitoring"""
292
- start_time = time.time()
293
-
294
- try:
295
- result = test_func(*args, **kwargs)
296
- duration = time.time() - start_time
297
- return result, duration, True
298
- except Exception as e:
299
- duration = time.time() - start_time
300
- print(f"⚠️ Performance test failed: {e}")
301
- return None, duration, False
302
-
303
- def run_all_tests(self, ddl: str = None, database: str = "DEMOBUILD",
304
- schema: str = None) -> Dict[str, Any]:
305
- """Run comprehensive test suite"""
306
-
307
- print("🧪 ENHANCED TEST FRAMEWORK")
308
- print("=" * 60)
309
- print(f"🚀 Starting comprehensive test suite at {datetime.now()}")
310
-
311
- self.start_time = time.time()
312
-
313
- # Auto-detect schema if not provided
314
- if not schema:
315
- # Look for recent schemas
316
- schema_pattern = datetime.now().strftime("2025%m%d")
317
- schema = f"{schema_pattern}_ENHANCED_TEST"
318
- print(f"📊 Using auto-generated schema: {schema}")
319
-
320
- # Default DDL for testing
321
- if not ddl:
322
- ddl = """
323
- CREATE TABLE CUSTOMERS (
324
- CUSTOMERID NUMBER(38,0) NOT NULL,
325
- FIRSTNAME VARCHAR(50) NOT NULL,
326
- LASTNAME VARCHAR(50) NOT NULL,
327
- EMAIL VARCHAR(100) NOT NULL
328
- );
329
-
330
- CREATE TABLE PRODUCTS (
331
- PRODUCTID NUMBER(38,0) NOT NULL,
332
- PRODUCTNAME VARCHAR(100) NOT NULL,
333
- PRICE NUMBER(10,2) NOT NULL
334
- );
335
- """
336
-
337
- # Run tests in order
338
- tests_to_run = [
339
- ("Environment", lambda: self.validate_environment()),
340
- ("Schema Validation", lambda: self.validate_schema_structure(ddl)),
341
- ("Snowflake Connection", lambda: self.test_snowflake_connection()),
342
- ("ThoughtSpot Auth", lambda: self.test_thoughtspot_auth()),
343
- ("Full Deployment", lambda: self.test_deployment_pipeline(ddl, database, schema))
344
- ]
345
-
346
- print("\n📋 Running Test Suite:")
347
-
348
- for test_name, test_func in tests_to_run:
349
- print(f"\n🔄 Running: {test_name}...")
350
-
351
- try:
352
- report = test_func()
353
- self.results.append(report)
354
-
355
- # Show immediate feedback
356
- duration_str = f"{report.duration:.2f}s"
357
- print(f" {report.result.value} {test_name} ({duration_str})")
358
-
359
- if report.error_message:
360
- print(f" 💬 {report.error_message}")
361
-
362
- except Exception as e:
363
- error_report = TestReport(
364
- test_name=test_name,
365
- result=TestResult.FAIL,
366
- duration=0.0,
367
- error_message=f"Test execution failed: {str(e)}"
368
- )
369
- self.results.append(error_report)
370
- print(f" ❌ FAIL {test_name} (execution error)")
371
-
372
- # Generate comprehensive report
373
- return self.generate_report()
374
-
375
- def generate_report(self) -> Dict[str, Any]:
376
- """Generate comprehensive test report"""
377
-
378
- total_duration = time.time() - self.start_time if self.start_time else 0
379
-
380
- # Calculate statistics
381
- stats = {
382
- "total_tests": len(self.results),
383
- "passed": len([r for r in self.results if r.result == TestResult.PASS]),
384
- "failed": len([r for r in self.results if r.result == TestResult.FAIL]),
385
- "warnings": len([r for r in self.results if r.result == TestResult.WARN]),
386
- "skipped": len([r for r in self.results if r.result == TestResult.SKIP]),
387
- "total_duration": total_duration,
388
- "average_duration": total_duration / len(self.results) if self.results else 0
389
- }
390
-
391
- success_rate = (stats["passed"] / stats["total_tests"] * 100) if stats["total_tests"] > 0 else 0
392
-
393
- # Print summary report
394
- print("\n" + "=" * 60)
395
- print("📊 TEST SUMMARY REPORT")
396
- print("=" * 60)
397
-
398
- print(f"🎯 Overall Success Rate: {success_rate:.1f}%")
399
- print(f"⏱️ Total Duration: {total_duration:.2f}s")
400
- print(f"📈 Tests: {stats['passed']}✅ {stats['failed']}❌ {stats['warnings']}⚠️ {stats['skipped']}⏭️")
401
-
402
- print(f"\n📋 Individual Results:")
403
- for result in self.results:
404
- duration_str = f"({result.duration:.2f}s)"
405
- print(f" {result.result.value} {result.test_name} {duration_str}")
406
- if result.error_message:
407
- print(f" 💬 {result.error_message}")
408
-
409
- # Recommendations
410
- print(f"\n🔧 RECOMMENDATIONS:")
411
- if stats["failed"] > 0:
412
- print(f" • Fix {stats['failed']} failing tests before deployment")
413
- if stats["warnings"] > 0:
414
- print(f" • Review {stats['warnings']} warnings for potential issues")
415
- if success_rate == 100:
416
- print(f" 🎉 All tests passing! System ready for deployment!")
417
-
418
- print("=" * 60)
419
-
420
- # Save detailed report
421
- report_data = {
422
- "timestamp": datetime.now().isoformat(),
423
- "statistics": stats,
424
- "success_rate": success_rate,
425
- "results": [asdict(r) for r in self.results]
426
- }
427
-
428
- # Save to file
429
- report_file = f"test_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
430
- with open(os.path.join("tests", report_file), "w") as f:
431
- json.dump(report_data, f, indent=2)
432
-
433
- print(f"📄 Detailed report saved: tests/{report_file}")
434
-
435
- return report_data
436
-
437
- def main():
438
- """Run enhanced test framework"""
439
- framework = TestFramework()
440
-
441
- # Example usage - can be customized
442
- results = framework.run_all_tests()
443
-
444
- return results["success_rate"] == 100.0
445
-
446
- if __name__ == "__main__":
447
- main()