dq-simulator / app.py
autodrive's picture
Rename streamlit_app.py to app.py
1b9893f verified
Raw History Blame Contribute Delete
50.9 kB
"""Streamlit UI for the Data Quality Issue Simulator.""" import io import sys import os import time # Ensure the simulator package is importable when run from this directory sys.path.insert(0, os.path.dirname(__file__)) import streamlit as st import pandas as pd import numpy as np from simulator import generator, injector, writer from simulator.issues import REGISTRY, list_issues # ── Page config ────────────────────────────────────────────────────────────── st.set_page_config( page_title="DQ Issue Simulator", page_icon="πŸ§ͺ", layout="wide", initial_sidebar_state="expanded", ) # ── Helpers ─────────────────────────────────────────────────────────────────── SEVERITY_COLOR = { "low": "#6c757d", "medium": "#fd7e14", "high": "#dc3545", "critical": "#6f42c1", } SEVERITY_EMOJI = {"low": "🟒", "medium": "🟑", "high": "πŸ”΄", "critical": "🟣"} CATEGORIES = sorted({i.category for i in REGISTRY.values()}) CATEGORY_LABELS = {c: c.replace("_", " ").title() for c in CATEGORIES} def severity_badge(sev: str) -> str: color = SEVERITY_COLOR.get(sev, "#888") return f'<span style="background:{color};color:#fff;padding:2px 8px;border-radius:4px;font-size:0.75rem;font-weight:600">{sev.upper()}</span>' # ── Bedrock / Claude helpers ────────────────────────────────────────────────── def _call_claude(prompt: str, model_id: str, region: str, max_tokens: int = 4096) -> str: """Call Claude via Anthropic API or Bedrock depending on selected provider.""" if ai_provider == "Anthropic API": try: import anthropic except ImportError: raise RuntimeError("Run: pip install anthropic") if not anthropic_api_key: raise RuntimeError("Enter your Anthropic API key in the sidebar.") client = anthropic.Anthropic(api_key=anthropic_api_key) msg = client.messages.create( model=anthropic_model, max_tokens=max_tokens, messages=[{"role": "user", "content": prompt}], ) return msg.content[0].text else: # AWS Bedrock import boto3, json kwargs = {"region_name": region} if aws_access_key and aws_secret_key: kwargs["aws_access_key_id"] = aws_access_key kwargs["aws_secret_access_key"] = aws_secret_key if aws_session_token: kwargs["aws_session_token"] = aws_session_token client = boto3.client("bedrock-runtime", **kwargs) body = json.dumps({ "anthropic_version": "bedrock-2023-05-31", "max_tokens": max_tokens, "messages": [{"role": "user", "content": prompt}], }) resp = client.invoke_model(modelId=model_id, body=body) return json.loads(resp["body"].read())["content"][0]["text"] def _issues_catalogue() -> str: """Compact text listing of all issues for prompt context.""" lines = [] for i in REGISTRY.values(): lines.append(f"{i.id} | {i.category} | {i.severity} | {i.description}") return "\n".join(lines) def ai_select_issues(scenario: str, model_id: str, region: str) -> list: """ Ask Claude to pick relevant issues and rates for a plain-English scenario. Returns list of {"id": str, "rate": float}. """ import json prompt = f"""You are a data quality expert. A user wants to simulate data quality issues in a user events dataset (clicks, purchases, logins, etc.) stored in AWS S3 / Glue. Available issues (id | category | severity | description): {_issues_catalogue()} User scenario: \"\"\"{scenario}\"\"\" Select the most relevant issues for this scenario. Return ONLY a valid JSON array β€” no explanation, no markdown, no code fences: [{{"id": "issue_id", "rate": 0.05}}, ...] Rules: - rate must be between 0.001 and 0.50 (fraction of rows affected) - Pick between 5 and 30 issues depending on scenario complexity - Set higher rates for issues the user explicitly emphasised - Never include issues unrelated to the scenario""" raw = _call_claude(prompt, model_id, region, max_tokens=1024) # Strip any accidental markdown raw = raw.strip().lstrip("```json").lstrip("```").rstrip("```").strip() return json.loads(raw) def ai_dq_report(applied: list, df_stats: dict, model_id: str, region: str) -> str: """Ask Claude to write a DQ report based on what was injected.""" applied_text = "\n".join( f"- {iid} [{REGISTRY[iid].severity}]: {REGISTRY[iid].description}" for iid in applied if iid in REGISTRY ) nulls_text = "\n".join( f" {col}: {pct:.1f}% null" for col, pct in df_stats.get("null_pct", {}).items() if pct > 0 ) prompt = f"""You are a senior data quality analyst reviewing a synthetic dataset generated to test a data pipeline for user events. The following {len(applied)} data quality issues were injected: {applied_text} Dataset shape: {df_stats['rows']:,} rows Γ— {df_stats['cols']} columns Columns with nulls: {nulls_text or ' (none)'} Write a concise data quality report with these sections: 1. **Executive Summary** (2 sentences) 2. **Critical Issues** β€” top 3 most impactful issues and why they matter for pipelines 3. **Pipeline Impact** β€” which downstream systems / queries are most at risk 4. **Recommended Tests** β€” bullet list of specific assertions to validate Keep it practical, concrete, and under 400 words.""" return _call_claude(prompt, model_id, region, max_tokens=1024) # ── Run log helpers ─────────────────────────────────────────────────────────── RUNS_LOG_PATH = os.path.join(os.path.dirname(__file__), "output", "runs_log.json") def _load_runs() -> list: if not os.path.exists(RUNS_LOG_PATH): return [] import json with open(RUNS_LOG_PATH) as f: return json.load(f) def _save_run(run: dict): import json os.makedirs(os.path.dirname(RUNS_LOG_PATH), exist_ok=True) runs = _load_runs() runs.append(run) with open(RUNS_LOG_PATH, "w") as f: json.dump(runs, f, indent=2, default=str) def _build_run_record(applied, skipped, df, output_path, exec_mode): import uuid as _uuid from datetime import datetime null_pct = { col: round(df[col].isnull().sum() / len(df) * 100, 2) for col in df.columns if df[col].isnull().sum() > 0 } by_cat = {} by_sev = {} for iid in applied: if iid in REGISTRY: issue = REGISTRY[iid] by_cat[issue.category] = by_cat.get(issue.category, 0) + 1 by_sev[issue.severity] = by_sev.get(issue.severity, 0) + 1 return { "run_id": str(_uuid.uuid4())[:8], "timestamp": datetime.now().isoformat(), "execution_mode": exec_mode, "rows_generated": len(df), "columns": df.shape[1], "issues_applied": applied, "issues_skipped": skipped, "issues_by_category": by_cat, "issues_by_severity": by_sev, "null_pct": null_pct, "output_path": output_path, } def ai_executive_report(runs: list, model_id: str, region: str) -> str: """Ask Claude to write a leadership-level executive report across all runs.""" import json from collections import Counter from datetime import datetime total = len(runs) if total == 0: return "No simulation runs recorded yet." # Aggregate statistics all_applied = [iid for r in runs for iid in r.get("issues_applied", [])] issue_counts = Counter(all_applied) cat_counts = Counter() sev_counts = Counter() for r in runs: for cat, n in r.get("issues_by_category", {}).items(): cat_counts[cat] += 1 # runs that included this category for sev, n in r.get("issues_by_severity", {}).items(): sev_counts[sev] += 1 # Top 10 most common issues as percentage of runs top_issues = [ f"{iid}: {count}/{total} runs ({count/total*100:.0f}%)" for iid, count in issue_counts.most_common(10) ] # Category prevalence cat_lines = [ f"{cat}: present in {count}/{total} runs ({count/total*100:.0f}%)" for cat, count in sorted(cat_counts.items(), key=lambda x: -x[1]) ] # Date range timestamps = [r["timestamp"] for r in runs if "timestamp" in r] date_range = f"{timestamps[0][:10]} to {timestamps[-1][:10]}" if timestamps else "unknown" avg_rows = int(sum(r.get("rows_generated", 0) for r in runs) / total) total_critical = sum(r.get("issues_by_severity", {}).get("critical", 0) for r in runs) prompt = f"""You are writing an executive report for data engineering leadership. This report summarises {total} data quality simulation runs conducted between {date_range}. KEY STATISTICS: - Total simulation runs: {total} - Average rows per simulation: {avg_rows:,} - Total critical-severity issues injected across all runs: {total_critical} ISSUE CATEGORY PREVALENCE (% of runs containing each category): {chr(10).join(cat_lines)} TOP 10 MOST FREQUENTLY SIMULATED ISSUES: {chr(10).join(top_issues)} SEVERITY DISTRIBUTION across runs: {json.dumps(dict(sev_counts), indent=2)} Write a professional executive report with these sections: 1. **Executive Summary** (3-4 sentences, leadership-friendly, no jargon) 2. **Key Findings** (bullet points, use X% of Y simulations language) 3. **Highest Risk Areas** (top 3 categories by prevalence, business impact) 4. **Recommended Actions** (3-5 concrete steps for the data engineering team) 5. **Next Steps** (what to prioritise in the next simulation cycle) Tone: confident, data-driven, suitable for a VP or CTO. Use percentages throughout.""" return _call_claude(prompt, model_id, region, max_tokens=2000) def _render_pdf(report_text: str, total_runs: int) -> bytes: """Render the executive report markdown as a PDF.""" from fpdf import FPDF from datetime import datetime pdf = FPDF() pdf.set_auto_page_break(auto=True, margin=20) pdf.add_page() # Header pdf.set_font("Helvetica", "B", 20) pdf.set_text_color(30, 30, 30) pdf.cell(0, 12, "Data Quality Simulation", ln=True) pdf.set_font("Helvetica", "B", 16) pdf.set_text_color(80, 80, 80) pdf.cell(0, 8, "Executive Report", ln=True) pdf.set_font("Helvetica", "", 10) pdf.set_text_color(120, 120, 120) pdf.cell(0, 6, f"Generated {datetime.now().strftime('%B %d, %Y')} Β· Based on {total_runs} simulation runs", ln=True) pdf.ln(4) pdf.set_draw_color(200, 200, 200) pdf.line(10, pdf.get_y(), 200, pdf.get_y()) pdf.ln(6) # Body β€” parse simple markdown pdf.set_font("Helvetica", "", 11) pdf.set_text_color(30, 30, 30) for line in report_text.split("\n"): line = line.rstrip() if line.startswith("## ") or line.startswith("# "): pdf.ln(4) pdf.set_font("Helvetica", "B", 13) pdf.set_text_color(20, 60, 120) pdf.multi_cell(0, 7, line.lstrip("# ").lstrip("* ")) pdf.set_font("Helvetica", "", 11) pdf.set_text_color(30, 30, 30) elif line.startswith("**") and line.endswith("**"): pdf.ln(3) pdf.set_font("Helvetica", "B", 12) pdf.set_text_color(20, 60, 120) pdf.multi_cell(0, 7, line.strip("*")) pdf.set_font("Helvetica", "", 11) pdf.set_text_color(30, 30, 30) elif line.startswith("- ") or line.startswith("* "): pdf.set_x(16) pdf.multi_cell(0, 6, "β€’ " + line[2:]) elif line == "": pdf.ln(3) else: # Strip inline ** bold markers clean = line.replace("**", "") pdf.multi_cell(0, 6, clean) return pdf.output() def ai_create_issue(description: str, model_id: str, region: str) -> str: """ Ask Claude to generate a new Issue definition in Python from a plain-English description. Returns the raw Python source code string. """ existing_ids = ", ".join(list(REGISTRY.keys())[:20]) + " ..." prompt = f"""You are a Python data quality engineer. Generate a new data quality issue for a pandas DataFrame of user events with these columns: event_id, user_id, session_id, event_type, timestamp, page_url, referrer_url, ip_address, device_type, os, browser, country_code, city, duration_seconds, revenue, product_id, quantity, search_query, user_agent, is_authenticated, user_age, email, user_registration_date The issue inject function signature is: def inject(df: pd.DataFrame, rate: float, rng: np.random.Generator) -> pd.DataFrame Rules: - Use only pandas, numpy, uuid, datetime β€” no other imports - Use pick_rows(df, rate, rng) for sampling rows (already imported) - For appending rows use pd.concat([df, new_rows_df], ignore_index=True) - The id must be snake_case, unique, not in: {existing_ids} - severity must be one of: low, medium, high, critical - default_rate must be between 0.001 and 0.5 User description: "{description}" Return ONLY a self-contained Python code block β€” no markdown, no explanation: import uuid import numpy as np import pandas as pd from .base import Issue, pick_rows def _<function_name>(df, rate, rng): # implementation ... ISSUES = [ Issue( id="<snake_case_id>", name="<Human readable name>", category="<one of: nulls/duplicates/type_mismatch/format/range/business_logic/encoding/statistical/structural>", description="<one sentence>", severity="<low|medium|high|critical>", default_rate=<float>, inject=_<function_name>, ), ]""" return _call_claude(prompt, model_id, region, max_tokens=1500) def ai_explain_issue(issue_id: str, model_id: str, region: str) -> str: """Ask Claude to explain a single DQ issue in plain English.""" issue = REGISTRY[issue_id] prompt = f"""Explain this data quality issue to a data engineer in 3-4 sentences. Include: what it is, a concrete real-world example of how it happens, and what breaks downstream. Issue: {issue.name} Category: {issue.category} Severity: {issue.severity} Description: {issue.description} Be concise and practical.""" return _call_claude(prompt, model_id, region, max_tokens=256) # ── EMR step submission ─────────────────────────────────────────────────────── def _run_emr_job(issue_ids, n_rows, rate_override, output_path, glue_database, glue_table, region, cluster_id, script_s3, py_files_s3, n_partitions, seed, extra_spark_conf): """Submit a spark-submit step to an existing running EMR cluster.""" try: import boto3 emr = boto3.client("emr", region_name=region) except ImportError: st.error("boto3 not installed. Run: pip install boto3") return issues_arg = ",".join(issue_ids) if issue_ids else "all" # Build spark-submit command spark_args = [ "spark-submit", "--deploy-mode", "cluster", "--conf", "spark.sql.adaptive.enabled=true", "--conf", "spark.sql.adaptive.coalescePartitions.enabled=true", "--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer", ] if py_files_s3: spark_args += ["--py-files", py_files_s3] for conf_pair in [c.strip() for c in extra_spark_conf.split(",") if c.strip()]: spark_args += ["--conf", conf_pair] spark_args += [ script_s3, "--rows", str(n_rows), "--issues", issues_arg, "--output_path", output_path, "--glue_table", glue_table or "user_events", "--region", region or "us-east-1", "--n_partitions", str(n_partitions), "--seed", str(seed), ] if glue_database: spark_args += ["--glue_database", glue_database] if rate_override is not None: spark_args += ["--rate", str(rate_override)] with st.spinner("Submitting EMR step..."): response = emr.add_job_flow_steps( JobFlowId=cluster_id, Steps=[{ "Name": "dq-simulator", "ActionOnFailure": "CONTINUE", "HadoopJarStep": { "Jar": "command-runner.jar", "Args": spark_args, }, }], ) step_id = response["StepIds"][0] st.success("βœ… EMR step submitted!") st.code( f"Cluster : {cluster_id}\n" f"Step ID : {step_id}\n" f"Rows : {n_rows:,}\n" f"Output : {output_path}" ) # ── Live status polling ─────────────────────────────────────── status_ph = st.empty() progress_bar = st.progress(0) # EMR step states: PENDING β†’ RUNNING β†’ COMPLETED / FAILED / CANCELLED / INTERRUPTED terminal_states = {"COMPLETED", "FAILED", "CANCELLED", "INTERRUPTED"} with st.spinner("Waiting for step to complete (polling every 15s)..."): attempt = 0 state = "PENDING" while state not in terminal_states: time.sleep(15) step_info = emr.describe_step(ClusterId=cluster_id, StepId=step_id)["Step"] state = step_info["Status"]["State"] timeline = step_info["Status"].get("Timeline", {}) started = timeline.get("StartDateTime") elapsed = ( int((time.time() - started.timestamp())) if started else 0 ) status_ph.info( f"**Step:** `{step_id}` | " f"**State:** `{state}` | " f"**Elapsed:** {elapsed}s" ) progress_bar.progress(min(0.95, 0.05 + attempt * 0.03)) attempt += 1 progress_bar.progress(1.0) if state == "COMPLETED": st.success(f"πŸŽ‰ Step completed in {elapsed}s | Output: `{output_path}`") if glue_database: st.info(f"Glue catalog updated: `{glue_database}.{glue_table}`") else: reason = step_info["Status"].get("FailureDetails", {}).get("Reason", "No details") st.error(f"Step {state}: {reason}") # ── Sidebar ─────────────────────────────────────────────────────────────────── with st.sidebar: st.title("πŸ§ͺ DQ Simulator") st.caption("Configure and generate dirty data") st.divider() # ── Execution mode ──────────────────────────────────────────── st.subheader("πŸ€– AI") ai_provider = st.radio("Provider", ["Anthropic API", "AWS Bedrock"], help="Anthropic API = direct, no AWS needed. Bedrock = stays inside AWS.") # ── Anthropic API ────────────────────────────────────────── anthropic_api_key = None anthropic_model = None if ai_provider == "Anthropic API": anthropic_api_key = st.text_input("Anthropic API key", type="password", help="Get one at console.anthropic.com") anthropic_model = st.selectbox("Model", [ "claude-sonnet-4-6", "claude-opus-4-6", "claude-haiku-4-5-20251001", ]) # ── Bedrock ──────────────────────────────────────────────── bedrock_region = "us-east-1" aws_access_key = aws_secret_key = aws_session_token = None bedrock_model = None if ai_provider == "AWS Bedrock": bedrock_model = st.selectbox("Claude model", [ "us.anthropic.claude-3-5-sonnet-20241022-v2:0", "us.anthropic.claude-3-5-haiku-20241022-v1:0", "us.anthropic.claude-3-sonnet-20240229-v1:0", ], help="Sonnet = smarter, Haiku = faster/cheaper") bedrock_region = st.text_input("Bedrock region", value="us-east-1") aws_cred_mode = st.radio("AWS credentials", ["Environment / IAM role", "Enter manually"]) if aws_cred_mode == "Enter manually": aws_access_key = st.text_input("Access Key ID", type="password") aws_secret_key = st.text_input("Secret Access Key", type="password") aws_session_token = st.text_input("Session Token (optional)", type="password") or None bedrock_enabled = st.toggle("Enable AI features", value=True) st.divider() exec_mode = st.radio( "⚑ Execution mode", ["Local (pandas)", "AWS EMR (PySpark)"], help="Local runs in-browser. EMR submits a Spark step to your running cluster.", ) is_glue = exec_mode == "AWS EMR (PySpark)" st.divider() st.subheader("βš™οΈ Data Settings") if is_glue: n_rows = st.number_input("Base rows to generate", min_value=1_000_000, max_value=2_000_000_000, value=200_000_000, step=1_000_000, help="200M is the default production scale") n_partitions = st.slider("Output Parquet files", 50, 2000, 200, step=50, help="~1M rows/file is optimal for Athena") else: n_rows = st.slider("Base rows to generate", 1_000, 500_000, 50_000, step=1_000, help="Number of clean rows before issue injection") n_partitions = 1 # unused for local seed = st.number_input("Random seed", value=42, step=1, help="Set a seed for reproducible output") st.divider() st.subheader("πŸ’‰ Injection Settings") rate_mode = st.radio("Injection rate mode", ["Per-issue defaults", "Global override"], help="Use each issue's built-in rate, or override all with one value") global_rate = None if rate_mode == "Global override": global_rate = st.slider("Injection rate", 0.001, 0.50, 0.02, step=0.001, format="%.3f", help="Fraction of rows each issue will affect") st.divider() st.subheader("πŸ’Ύ Output") if is_glue: output_path = st.text_input("S3 output path", value="s3://my-bucket/user_events/") glue_db = st.text_input("Glue database (optional)", value="my_database", help="If set, updates the Glue catalog after the step completes") glue_table = st.text_input("Glue table", value="user_events") aws_region = st.text_input("AWS region", value="us-east-1") st.divider() st.subheader("πŸ–₯️ EMR Cluster") emr_cluster_id = st.text_input("Cluster ID", value="j-XXXXXXXXXXXXX", help="The ID of your running EMR cluster") emr_script_s3 = st.text_input("Script S3 path", value="s3://my-bucket/scripts/job.py", help="Upload glue/job.py to S3 and paste the URI here") emr_py_files = st.text_input("--py-files S3 path (optional)", value="s3://my-bucket/scripts/glue.zip", help="Zip of the glue/ package. Run: zip -r glue.zip glue/") emr_spark_conf = st.text_input("Extra Spark --conf (comma-separated)", value="spark.executor.memory=8g,spark.executor.cores=4", help="Override any spark config key=value pairs") else: output_dest = st.radio("Destination", ["Local file", "S3"]) output_path = st.text_input( "Output path", value="./output/user_events.parquet" if output_dest == "Local file" else "s3://my-bucket/user_events/", ) if output_dest == "S3": glue_db = st.text_input("Glue database (optional)", value="") glue_table = st.text_input("Glue table", value="user_events") aws_region = st.text_input("AWS region", value="us-east-1") else: glue_db = glue_table = aws_region = None emr_cluster_id = emr_script_s3 = emr_py_files = emr_spark_conf = None # ── Main area ───────────────────────────────────────────────────────────────── st.title("Data Quality Issue Simulator") st.caption("103 issues across 9 categories Β· Parquet output Β· runs locally or on AWS EMR") page_sim, page_report = st.tabs(["πŸ§ͺ Simulator", "πŸ“Š Executive Report"]) # ── Executive Report page ───────────────────────────────────────────────────── with page_report: runs = _load_runs() total_runs = len(runs) r1, r2, r3, r4 = st.columns(4) r1.metric("Total runs logged", total_runs) if runs: from collections import Counter all_applied = [iid for r in runs for iid in r.get("issues_applied", [])] sev_critical = sum(r.get("issues_by_severity", {}).get("critical", 0) for r in runs) avg_rows = int(sum(r.get("rows_generated", 0) for r in runs) / total_runs) r2.metric("Avg rows/run", f"{avg_rows:,}") r3.metric("Critical issues (total)", sev_critical) r4.metric("Unique issues used", len(set(all_applied))) st.divider() # Run history table with st.expander("πŸ“‹ Run history", expanded=False): hist = [{ "Run ID": r["run_id"], "Date": r["timestamp"][:19], "Mode": r.get("execution_mode", "local"), "Rows": f'{r.get("rows_generated",0):,}', "Applied": len(r.get("issues_applied", [])), "Skipped": len(r.get("issues_skipped", [])), } for r in reversed(runs)] st.dataframe(hist, use_container_width=True) st.divider() rep_col1, rep_col2 = st.columns([2, 8]) with rep_col1: gen_report_btn = st.button("πŸ€– Generate Report with Claude", type="primary", use_container_width=True, disabled=not bedrock_enabled) if gen_report_btn: with st.spinner("Claude is writing the executive report..."): try: report_md = ai_executive_report( runs, bedrock_model or anthropic_model, bedrock_region, ) st.session_state["exec_report_md"] = report_md except Exception as e: st.error(f"Error: {e}") if "exec_report_md" in st.session_state: st.markdown(st.session_state["exec_report_md"]) st.divider() dl1, dl2, _ = st.columns([2, 2, 6]) with dl1: st.download_button( "⬇️ Download PDF", data=_render_pdf(st.session_state["exec_report_md"], total_runs), file_name="dq_executive_report.pdf", mime="application/pdf", use_container_width=True, ) with dl2: st.download_button( "⬇️ Download Markdown", data=st.session_state["exec_report_md"], file_name="dq_executive_report.md", mime="text/markdown", use_container_width=True, ) else: st.info("No simulation runs yet. Run at least one simulation in the **πŸ§ͺ Simulator** tab to generate a report.") # ── Simulator page ──────────────────────────────────────────────────────────── with page_sim: # ── Issue selector ──────────────────────────────────────────────────────────── # ── AI Issue Configurator ───────────────────────────────────────────────────── if bedrock_enabled: with st.expander("πŸ€– AI Issue Configurator β€” describe your scenario in plain English", expanded=False): scenario_input = st.text_area( "Describe the data quality problems you want to simulate", placeholder=( "e.g. 'Our pipeline sometimes receives duplicate events from mobile SDKs, " "occasional bot traffic spikes, and purchases with missing product IDs. " "We also see timestamp drift where events arrive out of order.'" ), height=100, ) ai_col1, ai_col2 = st.columns([2, 8]) with ai_col1: ai_configure = st.button("πŸ€– Configure with AI", type="secondary", disabled=not scenario_input.strip(), use_container_width=True) if ai_configure and scenario_input.strip(): with st.spinner("Claude is selecting issues..."): try: selections = ai_select_issues(scenario_input, bedrock_model or anthropic_model, bedrock_region) # Apply selections to session state ai_ids = {s["id"] for s in selections if s["id"] in REGISTRY} st.session_state.selected = ai_ids # Store AI-suggested rates st.session_state.ai_rates = { s["id"]: s["rate"] for s in selections if s["id"] in REGISTRY } st.success(f"Claude selected **{len(ai_ids)} issues**. Scroll down to review and adjust.") st.rerun() except Exception as e: st.error(f"Bedrock error: {e}") # ── AI Issue Creator ────────────────────────────────────────────────────────── if bedrock_enabled: with st.expander("✨ AI Issue Creator β€” generate a brand new issue from a description", expanded=False): new_issue_desc = st.text_area( "Describe the new data quality issue", placeholder="e.g. 'Create an issue where bot traffic is created by 10 unique bots, each generating 100 events per minute'", height=80, key="new_issue_desc", ) create_col1, create_col2 = st.columns([2, 8]) with create_col1: create_btn = st.button("✨ Generate Issue", type="secondary", disabled=not new_issue_desc.strip(), use_container_width=True) if create_btn and new_issue_desc.strip(): with st.spinner("Claude is writing the issue code..."): try: generated_code = ai_create_issue(new_issue_desc, bedrock_model or anthropic_model, bedrock_region) st.session_state["pending_issue_code"] = generated_code st.session_state["pending_issue_desc"] = new_issue_desc except Exception as e: st.error(f"Error: {e}") if "pending_issue_code" in st.session_state: st.markdown("**Generated code β€” review before adding:**") edited_code = st.text_area("Edit if needed", value=st.session_state["pending_issue_code"], height=300, key="edited_issue_code") confirm_col1, confirm_col2, _ = st.columns([2, 2, 6]) with confirm_col1: if st.button("βœ… Add to Simulator", type="primary", use_container_width=True): try: # Validate the code compiles compile(edited_code, "<ai_issue>", "exec") # Append to custom.py custom_path = os.path.join(os.path.dirname(__file__), "simulator", "issues", "custom.py") with open(custom_path, "r") as f: current = f.read() # Strip the ISSUES = [] placeholder on first write if current.strip().endswith("ISSUES = []"): current = current.replace("ISSUES = []", "") # Extract just the function + Issue(...) block, append to file new_content = current.rstrip() + "\n\n" + edited_code.strip() + "\n" # Merge ISSUES lists β€” replace last ISSUES = [...] with combined with open(custom_path, "w") as f: f.write(new_content) # Reload the module and registry import importlib import simulator.issues.custom as custom_mod import simulator.issues.registry as reg_mod importlib.reload(custom_mod) importlib.reload(reg_mod) # Re-import updated REGISTRY into app scope from simulator.issues.registry import REGISTRY as _NEW_REG REGISTRY.clear() REGISTRY.update(_NEW_REG) del st.session_state["pending_issue_code"] del st.session_state["pending_issue_desc"] st.success("Issue added! It will appear in the list below.") st.rerun() except SyntaxError as e: st.error(f"Syntax error in generated code: {e}") except Exception as e: st.error(f"Failed to add issue: {e}") with confirm_col2: if st.button("πŸ—‘οΈ Discard", use_container_width=True): del st.session_state["pending_issue_code"] st.rerun() st.subheader("Select Issues to Inject") # Global filter row col_sev, col_cat, col_search, col_all, col_none = st.columns([2, 2, 3, 1, 1]) with col_sev: sev_filter = st.selectbox("Severity filter", ["All", "critical", "high", "medium", "low"]) with col_cat: cat_filter = st.selectbox("Category filter", ["All"] + [CATEGORY_LABELS[c] for c in CATEGORIES]) with col_search: search = st.text_input("Search issues", placeholder="e.g. null, duplicate, timestamp ...") with col_all: st.write("") select_all = st.button("βœ… All", use_container_width=True) with col_none: st.write("") select_none = st.button("☐ None", use_container_width=True) # Build filtered issue list all_issues = list(REGISTRY.values()) filtered = all_issues if sev_filter != "All": filtered = [i for i in filtered if i.severity == sev_filter] if cat_filter != "All": cat_key = cat_filter.lower().replace(" ", "_") filtered = [i for i in filtered if i.category == cat_key] if search: q = search.lower() filtered = [i for i in filtered if q in i.id or q in i.description.lower()] # Track selected issues in session state if "selected" not in st.session_state: st.session_state.selected = {i.id for i in all_issues} if "ai_rates" not in st.session_state: st.session_state.ai_rates = {} if select_all: st.session_state.selected = {i.id for i in filtered} | st.session_state.selected if select_none: st.session_state.selected -= {i.id for i in filtered} # Render issues grouped by category filtered_by_cat = {} for issue in filtered: filtered_by_cat.setdefault(issue.category, []).append(issue) selected_ids = set(st.session_state.selected) rate_overrides = {} for cat in sorted(filtered_by_cat.keys()): cat_issues = filtered_by_cat[cat] cat_label = CATEGORY_LABELS[cat] n_selected = sum(1 for i in cat_issues if i.id in selected_ids) with st.expander(f"**{cat_label}** β€” {n_selected}/{len(cat_issues)} selected", expanded=True): # Category-level select all / none c1, c2, _ = st.columns([1, 1, 8]) with c1: if st.button("All", key=f"all_{cat}", use_container_width=True): for i in cat_issues: st.session_state.selected.add(i.id) st.rerun() with c2: if st.button("None", key=f"none_{cat}", use_container_width=True): for i in cat_issues: st.session_state.selected.discard(i.id) st.rerun() for issue in cat_issues: cols = st.columns([0.5, 3.2, 1.5, 1.5, 2, 0.8]) with cols[0]: checked = st.checkbox( "Select", value=(issue.id in st.session_state.selected), key=f"cb_{issue.id}", label_visibility="collapsed", ) if checked: st.session_state.selected.add(issue.id) else: st.session_state.selected.discard(issue.id) with cols[1]: ai_tag = " πŸ€–" if issue.id in st.session_state.ai_rates else "" st.markdown(f"`{issue.id}`{ai_tag} \n<small>{issue.description}</small>", unsafe_allow_html=True) with cols[2]: st.markdown(severity_badge(issue.severity), unsafe_allow_html=True) with cols[3]: st.caption(f"Default rate: {issue.default_rate:.1%}") with cols[4]: if rate_mode == "Per-issue defaults": ai_rate = st.session_state.ai_rates.get(issue.id, issue.default_rate) custom = st.slider( "rate", 0.001, 0.50, float(ai_rate), step=0.001, format="%.3f", key=f"rate_{issue.id}", label_visibility="collapsed", disabled=(issue.id not in st.session_state.selected), ) rate_overrides[issue.id] = custom else: st.caption("← global rate") with cols[5]: if bedrock_enabled and st.button("πŸ’¬", key=f"explain_{issue.id}", help="Ask Claude to explain this issue"): with st.spinner("Asking Claude..."): try: explanation = ai_explain_issue(issue.id, bedrock_model or anthropic_model, bedrock_region) st.session_state[f"explain_text_{issue.id}"] = explanation except Exception as e: st.session_state[f"explain_text_{issue.id}"] = f"Error: {e}" # Show explanation inline if available if f"explain_text_{issue.id}" in st.session_state: st.info(f"πŸ’¬ **{issue.name}:** {st.session_state[f'explain_text_{issue.id}']}") # ── Summary bar ────────────────────────────────────────────────────────────── st.divider() active = [i for i in all_issues if i.id in st.session_state.selected] by_sev = {} for i in active: by_sev[i.severity] = by_sev.get(i.severity, 0) + 1 summary_cols = st.columns(6) with summary_cols[0]: st.metric("Issues selected", len(active)) with summary_cols[1]: st.metric("🟣 Critical", by_sev.get("critical", 0)) with summary_cols[2]: st.metric("πŸ”΄ High", by_sev.get("high", 0)) with summary_cols[3]: st.metric("🟑 Medium", by_sev.get("medium", 0)) with summary_cols[4]: st.metric("🟒 Low", by_sev.get("low", 0)) with summary_cols[5]: st.metric("Base rows", f"{n_rows:,}") # ── Generate button ─────────────────────────────────────────────────────────── st.divider() run_col, _ = st.columns([2, 8]) with run_col: btn_label = "☁️ Submit Glue Job" if is_glue else "πŸš€ Generate & Inject" run = st.button(btn_label, type="primary", use_container_width=True, disabled=(len(active) == 0)) if run and is_glue: _run_emr_job( issue_ids=[i.id for i in active], n_rows=int(n_rows), rate_override=global_rate, output_path=output_path, glue_database=glue_db, glue_table=glue_table, region=aws_region, cluster_id=emr_cluster_id, script_s3=emr_script_s3, py_files_s3=emr_py_files or None, n_partitions=n_partitions, seed=int(seed), extra_spark_conf=emr_spark_conf or "", ) if run and not is_glue: progress = st.progress(0, text="Generating clean data...") log_area = st.empty() logs = [] # 1. Generate with st.spinner(f"Generating {n_rows:,} rows..."): df = generator.generate(n_rows, seed=int(seed)) progress.progress(20, text=f"Generated {n_rows:,} clean rows") # 2. Inject issue_ids = [i.id for i in active] # Build per-issue rates if in per-issue mode if rate_mode == "Per-issue defaults": # Temporarily patch each issue's default_rate originals = {} for iid in issue_ids: originals[iid] = REGISTRY[iid].default_rate REGISTRY[iid].default_rate = rate_overrides.get(iid, REGISTRY[iid].default_rate) rate_arg = None else: originals = {} rate_arg = global_rate applied = [] skipped = [] rng = np.random.default_rng(int(seed)) progress.progress(25, text="Injecting issues...") for idx, iid in enumerate(issue_ids): issue = REGISTRY[iid] rate = rate_arg if rate_arg is not None else issue.default_rate try: df = issue.inject(df, rate, rng) applied.append(iid) logs.append(f"βœ… {iid}") except Exception as e: skipped.append(iid) logs.append(f"⚠️ {iid} β€” skipped ({e})") pct = 25 + int(70 * (idx + 1) / len(issue_ids)) progress.progress(pct, text=f"Injecting: {iid}") log_area.text("\n".join(logs[-20:])) # Restore original rates for iid, orig in originals.items(): REGISTRY[iid].default_rate = orig # 3. Write progress.progress(96, text="Writing parquet...") try: out = writer.write( df, output=output_path, glue_database=glue_db or None, glue_table=glue_table or "user_events", region=aws_region or "us-east-1", ) write_ok = True write_msg = f"Written to `{out}`" except Exception as e: write_ok = False write_msg = f"Write failed: {e}" progress.progress(100, text="Done!") log_area.empty() # ── Save run to log ─────────────────────────────────────────────────────── _save_run(_build_run_record(applied, skipped, df, output_path, exec_mode)) # ── Results ─────────────────────────────────────────────────────────────── st.success(f"Done! **{len(applied)}** issues injected Β· **{df.shape[0]:,}** rows Β· **{df.shape[1]}** columns") if skipped: st.warning(f"{len(skipped)} issues skipped: {', '.join(skipped)}") if write_ok: st.info(write_msg) else: st.error(write_msg) # Stats st.subheader("Results") r1, r2, r3, r4 = st.columns(4) r1.metric("Total rows", f"{df.shape[0]:,}", delta=f"+{df.shape[0]-n_rows:,} from injection") r2.metric("Columns", df.shape[1]) r3.metric("Issues applied", len(applied)) r4.metric("Issues skipped", len(skipped)) # Tabs: preview, nulls, types, log tab_preview, tab_nulls, tab_types, tab_log, tab_report, tab_download = st.tabs( ["πŸ“‹ Preview", "πŸ•³οΈ Null Analysis", "πŸ”¬ Type Analysis", "πŸ“ Injection Log", "πŸ€– AI DQ Report", "⬇️ Download"] ) with tab_preview: st.caption(f"First 200 rows of {df.shape[0]:,} total") st.dataframe(df.head(200), use_container_width=True, height=400) with tab_nulls: null_counts = df.isnull().sum().reset_index() null_counts.columns = ["column", "null_count"] null_counts["null_%"] = (null_counts["null_count"] / len(df) * 100).round(2) null_counts = null_counts[null_counts["null_count"] > 0].sort_values("null_count", ascending=False) if null_counts.empty: st.info("No null values found.") else: st.bar_chart(null_counts.set_index("column")["null_%"], use_container_width=True) st.dataframe(null_counts, use_container_width=True) with tab_types: type_df = pd.DataFrame({ "column": df.columns, "dtype": [str(df[c].dtype) for c in df.columns], "non_null": [df[c].count() for c in df.columns], "unique": [df[c].nunique() for c in df.columns], "sample_value": [str(df[c].dropna().iloc[0]) if df[c].count() > 0 else "N/A" for c in df.columns], }) st.dataframe(type_df, use_container_width=True) with tab_log: st.code("\n".join(logs)) with tab_report: if not bedrock_enabled: st.info("Enable AI features in the sidebar to generate a DQ report.") else: if st.button("πŸ€– Generate DQ Report with Claude", type="secondary"): null_pct = { col: round(df[col].isnull().sum() / len(df) * 100, 1) for col in df.columns } df_stats = { "rows": len(df), "cols": df.shape[1], "null_pct": null_pct, } with st.spinner("Claude is writing your DQ report..."): try: report = ai_dq_report(applied, df_stats, bedrock_model or anthropic_model, bedrock_region) st.session_state["last_dq_report"] = report except Exception as e: st.error(f"Bedrock error: {e}") if "last_dq_report" in st.session_state: st.markdown(st.session_state["last_dq_report"]) st.download_button( "⬇️ Download Report (.md)", data=st.session_state["last_dq_report"], file_name="dq_report.md", mime="text/markdown", ) with tab_download: buf = io.BytesIO() df_str = df.astype(str) try: import pyarrow as pa import pyarrow.parquet as pq safe = {} for col in df.columns: try: safe[col] = pa.array(df[col].tolist()) except Exception: safe[col] = pa.array(df[col].astype(str).tolist(), type=pa.string()) table = pa.table(safe) pq.write_table(table, buf) buf.seek(0) st.download_button( label="⬇️ Download as Parquet", data=buf, file_name="user_events_dq.parquet", mime="application/octet-stream", use_container_width=True, ) except Exception as e: st.error(f"Could not prepare download: {e}") csv_buf = io.StringIO() df.astype(str).to_csv(csv_buf, index=False) st.download_button( label="⬇️ Download as CSV", data=csv_buf.getvalue(), file_name="user_events_dq.csv", mime="text/csv", use_container_width=True, )
"""Streamlit UI for the Data Quality Issue Simulator."""
import io
import sys
import os
import time
# Ensure the simulator package is importable when run from this directory
sys.path.insert(0, os.path.dirname(__file__))
import streamlit as st
import pandas as pd
import numpy as np
from simulator import generator, injector, writer
from simulator.issues import REGISTRY, list_issues
# ── Page config ──────────────────────────────────────────────────────────────
st.set_page_config(
page_title="DQ Issue Simulator",
page_icon="πŸ§ͺ",
layout="wide",
initial_sidebar_state="expanded",
)
# ── Helpers ───────────────────────────────────────────────────────────────────
SEVERITY_COLOR = {
"low": "#6c757d",
"medium": "#fd7e14",
"high": "#dc3545",
"critical": "#6f42c1",
}
SEVERITY_EMOJI = {"low": "🟒", "medium": "🟑", "high": "πŸ”΄", "critical": "🟣"}
CATEGORIES = sorted({i.category for i in REGISTRY.values()})
CATEGORY_LABELS = {c: c.replace("_", " ").title() for c in CATEGORIES}
def severity_badge(sev: str) -> str:
color = SEVERITY_COLOR.get(sev, "#888")
return f'<span style="background:{color};color:#fff;padding:2px 8px;border-radius:4px;font-size:0.75rem;font-weight:600">{sev.upper()}</span>'
# ── Bedrock / Claude helpers ──────────────────────────────────────────────────
def _call_claude(prompt: str, model_id: str, region: str, max_tokens: int = 4096) -> str:
"""Call Claude via Anthropic API or Bedrock depending on selected provider."""
if ai_provider == "Anthropic API":
try:
import anthropic
except ImportError:
raise RuntimeError("Run: pip install anthropic")
if not anthropic_api_key:
raise RuntimeError("Enter your Anthropic API key in the sidebar.")
client = anthropic.Anthropic(api_key=anthropic_api_key)
msg = client.messages.create(
model=anthropic_model,
max_tokens=max_tokens,
messages=[{"role": "user", "content": prompt}],
)
return msg.content[0].text
else: # AWS Bedrock
import boto3, json
kwargs = {"region_name": region}
if aws_access_key and aws_secret_key:
kwargs["aws_access_key_id"] = aws_access_key
kwargs["aws_secret_access_key"] = aws_secret_key
if aws_session_token:
kwargs["aws_session_token"] = aws_session_token
client = boto3.client("bedrock-runtime", **kwargs)
body = json.dumps({
"anthropic_version": "bedrock-2023-05-31",
"max_tokens": max_tokens,
"messages": [{"role": "user", "content": prompt}],
})
resp = client.invoke_model(modelId=model_id, body=body)
return json.loads(resp["body"].read())["content"][0]["text"]
def _issues_catalogue() -> str:
"""Compact text listing of all issues for prompt context."""
lines = []
for i in REGISTRY.values():
lines.append(f"{i.id} | {i.category} | {i.severity} | {i.description}")
return "\n".join(lines)
def ai_select_issues(scenario: str, model_id: str, region: str) -> list:
"""
Ask Claude to pick relevant issues and rates for a plain-English scenario.
Returns list of {"id": str, "rate": float}.
"""
import json
prompt = f"""You are a data quality expert. A user wants to simulate data quality issues
in a user events dataset (clicks, purchases, logins, etc.) stored in AWS S3 / Glue.
Available issues (id | category | severity | description):
{_issues_catalogue()}
User scenario: