Spaces:
Runtime error
Runtime error
Download app.py from autodrive/dq-simulator: direct link, hf CLI and curl.
- Browser
- Download file 50.9 kB
-
https://huggingface.co/spaces/autodrive/dq-simulator/resolve/main/app.py
- Command line
-
hf download hf://spaces/autodrive/dq-simulator/app.py
-
curl -L -o app.py https://huggingface.co/spaces/autodrive/dq-simulator/resolve/main/app.py
50.9 kB
"""Streamlit UI for the Data Quality Issue Simulator."""
import io
import sys
import os
import time
# Ensure the simulator package is importable when run from this directory
sys.path.insert(0, os.path.dirname(__file__))
import streamlit as st
import pandas as pd
import numpy as np
from simulator import generator, injector, writer
from simulator.issues import REGISTRY, list_issues
# ββ Page config ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
st.set_page_config(
page_title="DQ Issue Simulator",
page_icon="π§ͺ",
layout="wide",
initial_sidebar_state="expanded",
)
# ββ Helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
SEVERITY_COLOR = {
"low": "#6c757d",
"medium": "#fd7e14",
"high": "#dc3545",
"critical": "#6f42c1",
}
SEVERITY_EMOJI = {"low": "π’", "medium": "π‘", "high": "π΄", "critical": "π£"}
CATEGORIES = sorted({i.category for i in REGISTRY.values()})
CATEGORY_LABELS = {c: c.replace("_", " ").title() for c in CATEGORIES}
def severity_badge(sev: str) -> str:
color = SEVERITY_COLOR.get(sev, "#888")
return f'<span style="background:{color};color:#fff;padding:2px 8px;border-radius:4px;font-size:0.75rem;font-weight:600">{sev.upper()}</span>'
# ββ Bedrock / Claude helpers ββββββββββββββββββββββββββββββββββββββββββββββββββ
def _call_claude(prompt: str, model_id: str, region: str, max_tokens: int = 4096) -> str:
"""Call Claude via Anthropic API or Bedrock depending on selected provider."""
if ai_provider == "Anthropic API":
try:
import anthropic
except ImportError:
raise RuntimeError("Run: pip install anthropic")
if not anthropic_api_key:
raise RuntimeError("Enter your Anthropic API key in the sidebar.")
client = anthropic.Anthropic(api_key=anthropic_api_key)
msg = client.messages.create(
model=anthropic_model,
max_tokens=max_tokens,
messages=[{"role": "user", "content": prompt}],
)
return msg.content[0].text
else: # AWS Bedrock
import boto3, json
kwargs = {"region_name": region}
if aws_access_key and aws_secret_key:
kwargs["aws_access_key_id"] = aws_access_key
kwargs["aws_secret_access_key"] = aws_secret_key
if aws_session_token:
kwargs["aws_session_token"] = aws_session_token
client = boto3.client("bedrock-runtime", **kwargs)
body = json.dumps({
"anthropic_version": "bedrock-2023-05-31",
"max_tokens": max_tokens,
"messages": [{"role": "user", "content": prompt}],
})
resp = client.invoke_model(modelId=model_id, body=body)
return json.loads(resp["body"].read())["content"][0]["text"]
def _issues_catalogue() -> str:
"""Compact text listing of all issues for prompt context."""
lines = []
for i in REGISTRY.values():
lines.append(f"{i.id} | {i.category} | {i.severity} | {i.description}")
return "\n".join(lines)
def ai_select_issues(scenario: str, model_id: str, region: str) -> list:
"""
Ask Claude to pick relevant issues and rates for a plain-English scenario.
Returns list of {"id": str, "rate": float}.
"""
import json
prompt = f"""You are a data quality expert. A user wants to simulate data quality issues
in a user events dataset (clicks, purchases, logins, etc.) stored in AWS S3 / Glue.
Available issues (id | category | severity | description):
{_issues_catalogue()}
User scenario:
\"\"\"{scenario}\"\"\"
Select the most relevant issues for this scenario.
Return ONLY a valid JSON array β no explanation, no markdown, no code fences:
[{{"id": "issue_id", "rate": 0.05}}, ...]
Rules:
- rate must be between 0.001 and 0.50 (fraction of rows affected)
- Pick between 5 and 30 issues depending on scenario complexity
- Set higher rates for issues the user explicitly emphasised
- Never include issues unrelated to the scenario"""
raw = _call_claude(prompt, model_id, region, max_tokens=1024)
# Strip any accidental markdown
raw = raw.strip().lstrip("```json").lstrip("```").rstrip("```").strip()
return json.loads(raw)
def ai_dq_report(applied: list, df_stats: dict, model_id: str, region: str) -> str:
"""Ask Claude to write a DQ report based on what was injected."""
applied_text = "\n".join(
f"- {iid} [{REGISTRY[iid].severity}]: {REGISTRY[iid].description}"
for iid in applied if iid in REGISTRY
)
nulls_text = "\n".join(
f" {col}: {pct:.1f}% null"
for col, pct in df_stats.get("null_pct", {}).items()
if pct > 0
)
prompt = f"""You are a senior data quality analyst reviewing a synthetic dataset
generated to test a data pipeline for user events.
The following {len(applied)} data quality issues were injected:
{applied_text}
Dataset shape: {df_stats['rows']:,} rows Γ {df_stats['cols']} columns
Columns with nulls:
{nulls_text or ' (none)'}
Write a concise data quality report with these sections:
1. **Executive Summary** (2 sentences)
2. **Critical Issues** β top 3 most impactful issues and why they matter for pipelines
3. **Pipeline Impact** β which downstream systems / queries are most at risk
4. **Recommended Tests** β bullet list of specific assertions to validate
Keep it practical, concrete, and under 400 words."""
return _call_claude(prompt, model_id, region, max_tokens=1024)
# ββ Run log helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
RUNS_LOG_PATH = os.path.join(os.path.dirname(__file__), "output", "runs_log.json")
def _load_runs() -> list:
if not os.path.exists(RUNS_LOG_PATH):
return []
import json
with open(RUNS_LOG_PATH) as f:
return json.load(f)
def _save_run(run: dict):
import json
os.makedirs(os.path.dirname(RUNS_LOG_PATH), exist_ok=True)
runs = _load_runs()
runs.append(run)
with open(RUNS_LOG_PATH, "w") as f:
json.dump(runs, f, indent=2, default=str)
def _build_run_record(applied, skipped, df, output_path, exec_mode):
import uuid as _uuid
from datetime import datetime
null_pct = {
col: round(df[col].isnull().sum() / len(df) * 100, 2)
for col in df.columns if df[col].isnull().sum() > 0
}
by_cat = {}
by_sev = {}
for iid in applied:
if iid in REGISTRY:
issue = REGISTRY[iid]
by_cat[issue.category] = by_cat.get(issue.category, 0) + 1
by_sev[issue.severity] = by_sev.get(issue.severity, 0) + 1
return {
"run_id": str(_uuid.uuid4())[:8],
"timestamp": datetime.now().isoformat(),
"execution_mode": exec_mode,
"rows_generated": len(df),
"columns": df.shape[1],
"issues_applied": applied,
"issues_skipped": skipped,
"issues_by_category": by_cat,
"issues_by_severity": by_sev,
"null_pct": null_pct,
"output_path": output_path,
}
def ai_executive_report(runs: list, model_id: str, region: str) -> str:
"""Ask Claude to write a leadership-level executive report across all runs."""
import json
from collections import Counter
from datetime import datetime
total = len(runs)
if total == 0:
return "No simulation runs recorded yet."
# Aggregate statistics
all_applied = [iid for r in runs for iid in r.get("issues_applied", [])]
issue_counts = Counter(all_applied)
cat_counts = Counter()
sev_counts = Counter()
for r in runs:
for cat, n in r.get("issues_by_category", {}).items():
cat_counts[cat] += 1 # runs that included this category
for sev, n in r.get("issues_by_severity", {}).items():
sev_counts[sev] += 1
# Top 10 most common issues as percentage of runs
top_issues = [
f"{iid}: {count}/{total} runs ({count/total*100:.0f}%)"
for iid, count in issue_counts.most_common(10)
]
# Category prevalence
cat_lines = [
f"{cat}: present in {count}/{total} runs ({count/total*100:.0f}%)"
for cat, count in sorted(cat_counts.items(), key=lambda x: -x[1])
]
# Date range
timestamps = [r["timestamp"] for r in runs if "timestamp" in r]
date_range = f"{timestamps[0][:10]} to {timestamps[-1][:10]}" if timestamps else "unknown"
avg_rows = int(sum(r.get("rows_generated", 0) for r in runs) / total)
total_critical = sum(r.get("issues_by_severity", {}).get("critical", 0) for r in runs)
prompt = f"""You are writing an executive report for data engineering leadership.
This report summarises {total} data quality simulation runs conducted between {date_range}.
KEY STATISTICS:
- Total simulation runs: {total}
- Average rows per simulation: {avg_rows:,}
- Total critical-severity issues injected across all runs: {total_critical}
ISSUE CATEGORY PREVALENCE (% of runs containing each category):
{chr(10).join(cat_lines)}
TOP 10 MOST FREQUENTLY SIMULATED ISSUES:
{chr(10).join(top_issues)}
SEVERITY DISTRIBUTION across runs:
{json.dumps(dict(sev_counts), indent=2)}
Write a professional executive report with these sections:
1. **Executive Summary** (3-4 sentences, leadership-friendly, no jargon)
2. **Key Findings** (bullet points, use X% of Y simulations language)
3. **Highest Risk Areas** (top 3 categories by prevalence, business impact)
4. **Recommended Actions** (3-5 concrete steps for the data engineering team)
5. **Next Steps** (what to prioritise in the next simulation cycle)
Tone: confident, data-driven, suitable for a VP or CTO. Use percentages throughout."""
return _call_claude(prompt, model_id, region, max_tokens=2000)
def _render_pdf(report_text: str, total_runs: int) -> bytes:
"""Render the executive report markdown as a PDF."""
from fpdf import FPDF
from datetime import datetime
pdf = FPDF()
pdf.set_auto_page_break(auto=True, margin=20)
pdf.add_page()
# Header
pdf.set_font("Helvetica", "B", 20)
pdf.set_text_color(30, 30, 30)
pdf.cell(0, 12, "Data Quality Simulation", ln=True)
pdf.set_font("Helvetica", "B", 16)
pdf.set_text_color(80, 80, 80)
pdf.cell(0, 8, "Executive Report", ln=True)
pdf.set_font("Helvetica", "", 10)
pdf.set_text_color(120, 120, 120)
pdf.cell(0, 6, f"Generated {datetime.now().strftime('%B %d, %Y')} Β· Based on {total_runs} simulation runs", ln=True)
pdf.ln(4)
pdf.set_draw_color(200, 200, 200)
pdf.line(10, pdf.get_y(), 200, pdf.get_y())
pdf.ln(6)
# Body β parse simple markdown
pdf.set_font("Helvetica", "", 11)
pdf.set_text_color(30, 30, 30)
for line in report_text.split("\n"):
line = line.rstrip()
if line.startswith("## ") or line.startswith("# "):
pdf.ln(4)
pdf.set_font("Helvetica", "B", 13)
pdf.set_text_color(20, 60, 120)
pdf.multi_cell(0, 7, line.lstrip("# ").lstrip("* "))
pdf.set_font("Helvetica", "", 11)
pdf.set_text_color(30, 30, 30)
elif line.startswith("**") and line.endswith("**"):
pdf.ln(3)
pdf.set_font("Helvetica", "B", 12)
pdf.set_text_color(20, 60, 120)
pdf.multi_cell(0, 7, line.strip("*"))
pdf.set_font("Helvetica", "", 11)
pdf.set_text_color(30, 30, 30)
elif line.startswith("- ") or line.startswith("* "):
pdf.set_x(16)
pdf.multi_cell(0, 6, "β’ " + line[2:])
elif line == "":
pdf.ln(3)
else:
# Strip inline ** bold markers
clean = line.replace("**", "")
pdf.multi_cell(0, 6, clean)
return pdf.output()
def ai_create_issue(description: str, model_id: str, region: str) -> str:
"""
Ask Claude to generate a new Issue definition in Python from a plain-English description.
Returns the raw Python source code string.
"""
existing_ids = ", ".join(list(REGISTRY.keys())[:20]) + " ..."
prompt = f"""You are a Python data quality engineer. Generate a new data quality issue
for a pandas DataFrame of user events with these columns:
event_id, user_id, session_id, event_type, timestamp, page_url, referrer_url,
ip_address, device_type, os, browser, country_code, city, duration_seconds,
revenue, product_id, quantity, search_query, user_agent, is_authenticated,
user_age, email, user_registration_date
The issue inject function signature is:
def inject(df: pd.DataFrame, rate: float, rng: np.random.Generator) -> pd.DataFrame
Rules:
- Use only pandas, numpy, uuid, datetime β no other imports
- Use pick_rows(df, rate, rng) for sampling rows (already imported)
- For appending rows use pd.concat([df, new_rows_df], ignore_index=True)
- The id must be snake_case, unique, not in: {existing_ids}
- severity must be one of: low, medium, high, critical
- default_rate must be between 0.001 and 0.5
User description: "{description}"
Return ONLY a self-contained Python code block β no markdown, no explanation:
import uuid
import numpy as np
import pandas as pd
from .base import Issue, pick_rows
def _<function_name>(df, rate, rng):
# implementation
...
ISSUES = [
Issue(
id="<snake_case_id>",
name="<Human readable name>",
category="<one of: nulls/duplicates/type_mismatch/format/range/business_logic/encoding/statistical/structural>",
description="<one sentence>",
severity="<low|medium|high|critical>",
default_rate=<float>,
inject=_<function_name>,
),
]"""
return _call_claude(prompt, model_id, region, max_tokens=1500)
def ai_explain_issue(issue_id: str, model_id: str, region: str) -> str:
"""Ask Claude to explain a single DQ issue in plain English."""
issue = REGISTRY[issue_id]
prompt = f"""Explain this data quality issue to a data engineer in 3-4 sentences.
Include: what it is, a concrete real-world example of how it happens, and what breaks downstream.
Issue: {issue.name}
Category: {issue.category}
Severity: {issue.severity}
Description: {issue.description}
Be concise and practical."""
return _call_claude(prompt, model_id, region, max_tokens=256)
# ββ EMR step submission βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
def _run_emr_job(issue_ids, n_rows, rate_override, output_path, glue_database,
glue_table, region, cluster_id, script_s3, py_files_s3,
n_partitions, seed, extra_spark_conf):
"""Submit a spark-submit step to an existing running EMR cluster."""
try:
import boto3
emr = boto3.client("emr", region_name=region)
except ImportError:
st.error("boto3 not installed. Run: pip install boto3")
return
issues_arg = ",".join(issue_ids) if issue_ids else "all"
# Build spark-submit command
spark_args = [
"spark-submit",
"--deploy-mode", "cluster",
"--conf", "spark.sql.adaptive.enabled=true",
"--conf", "spark.sql.adaptive.coalescePartitions.enabled=true",
"--conf", "spark.serializer=org.apache.spark.serializer.KryoSerializer",
]
if py_files_s3:
spark_args += ["--py-files", py_files_s3]
for conf_pair in [c.strip() for c in extra_spark_conf.split(",") if c.strip()]:
spark_args += ["--conf", conf_pair]
spark_args += [
script_s3,
"--rows", str(n_rows),
"--issues", issues_arg,
"--output_path", output_path,
"--glue_table", glue_table or "user_events",
"--region", region or "us-east-1",
"--n_partitions", str(n_partitions),
"--seed", str(seed),
]
if glue_database:
spark_args += ["--glue_database", glue_database]
if rate_override is not None:
spark_args += ["--rate", str(rate_override)]
with st.spinner("Submitting EMR step..."):
response = emr.add_job_flow_steps(
JobFlowId=cluster_id,
Steps=[{
"Name": "dq-simulator",
"ActionOnFailure": "CONTINUE",
"HadoopJarStep": {
"Jar": "command-runner.jar",
"Args": spark_args,
},
}],
)
step_id = response["StepIds"][0]
st.success("β
EMR step submitted!")
st.code(
f"Cluster : {cluster_id}\n"
f"Step ID : {step_id}\n"
f"Rows : {n_rows:,}\n"
f"Output : {output_path}"
)
# ββ Live status polling βββββββββββββββββββββββββββββββββββββββ
status_ph = st.empty()
progress_bar = st.progress(0)
# EMR step states: PENDING β RUNNING β COMPLETED / FAILED / CANCELLED / INTERRUPTED
terminal_states = {"COMPLETED", "FAILED", "CANCELLED", "INTERRUPTED"}
with st.spinner("Waiting for step to complete (polling every 15s)..."):
attempt = 0
state = "PENDING"
while state not in terminal_states:
time.sleep(15)
step_info = emr.describe_step(ClusterId=cluster_id, StepId=step_id)["Step"]
state = step_info["Status"]["State"]
timeline = step_info["Status"].get("Timeline", {})
started = timeline.get("StartDateTime")
elapsed = (
int((time.time() - started.timestamp())) if started else 0
)
status_ph.info(
f"**Step:** `{step_id}` | "
f"**State:** `{state}` | "
f"**Elapsed:** {elapsed}s"
)
progress_bar.progress(min(0.95, 0.05 + attempt * 0.03))
attempt += 1
progress_bar.progress(1.0)
if state == "COMPLETED":
st.success(f"π Step completed in {elapsed}s | Output: `{output_path}`")
if glue_database:
st.info(f"Glue catalog updated: `{glue_database}.{glue_table}`")
else:
reason = step_info["Status"].get("FailureDetails", {}).get("Reason", "No details")
st.error(f"Step {state}: {reason}")
# ββ Sidebar βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
with st.sidebar:
st.title("π§ͺ DQ Simulator")
st.caption("Configure and generate dirty data")
st.divider()
# ββ Execution mode ββββββββββββββββββββββββββββββββββββββββββββ
st.subheader("π€ AI")
ai_provider = st.radio("Provider", ["Anthropic API", "AWS Bedrock"],
help="Anthropic API = direct, no AWS needed. Bedrock = stays inside AWS.")
# ββ Anthropic API ββββββββββββββββββββββββββββββββββββββββββ
anthropic_api_key = None
anthropic_model = None
if ai_provider == "Anthropic API":
anthropic_api_key = st.text_input("Anthropic API key", type="password",
help="Get one at console.anthropic.com")
anthropic_model = st.selectbox("Model", [
"claude-sonnet-4-6",
"claude-opus-4-6",
"claude-haiku-4-5-20251001",
])
# ββ Bedrock ββββββββββββββββββββββββββββββββββββββββββββββββ
bedrock_region = "us-east-1"
aws_access_key = aws_secret_key = aws_session_token = None
bedrock_model = None
if ai_provider == "AWS Bedrock":
bedrock_model = st.selectbox("Claude model", [
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
"us.anthropic.claude-3-5-haiku-20241022-v1:0",
"us.anthropic.claude-3-sonnet-20240229-v1:0",
], help="Sonnet = smarter, Haiku = faster/cheaper")
bedrock_region = st.text_input("Bedrock region", value="us-east-1")
aws_cred_mode = st.radio("AWS credentials", ["Environment / IAM role", "Enter manually"])
if aws_cred_mode == "Enter manually":
aws_access_key = st.text_input("Access Key ID", type="password")
aws_secret_key = st.text_input("Secret Access Key", type="password")
aws_session_token = st.text_input("Session Token (optional)", type="password") or None
bedrock_enabled = st.toggle("Enable AI features", value=True)
st.divider()
exec_mode = st.radio(
"β‘ Execution mode",
["Local (pandas)", "AWS EMR (PySpark)"],
help="Local runs in-browser. EMR submits a Spark step to your running cluster.",
)
is_glue = exec_mode == "AWS EMR (PySpark)"
st.divider()
st.subheader("βοΈ Data Settings")
if is_glue:
n_rows = st.number_input("Base rows to generate", min_value=1_000_000,
max_value=2_000_000_000, value=200_000_000, step=1_000_000,
help="200M is the default production scale")
n_partitions = st.slider("Output Parquet files", 50, 2000, 200, step=50,
help="~1M rows/file is optimal for Athena")
else:
n_rows = st.slider("Base rows to generate", 1_000, 500_000, 50_000, step=1_000,
help="Number of clean rows before issue injection")
n_partitions = 1 # unused for local
seed = st.number_input("Random seed", value=42, step=1,
help="Set a seed for reproducible output")
st.divider()
st.subheader("π Injection Settings")
rate_mode = st.radio("Injection rate mode",
["Per-issue defaults", "Global override"],
help="Use each issue's built-in rate, or override all with one value")
global_rate = None
if rate_mode == "Global override":
global_rate = st.slider("Injection rate", 0.001, 0.50, 0.02, step=0.001,
format="%.3f",
help="Fraction of rows each issue will affect")
st.divider()
st.subheader("πΎ Output")
if is_glue:
output_path = st.text_input("S3 output path", value="s3://my-bucket/user_events/")
glue_db = st.text_input("Glue database (optional)", value="my_database",
help="If set, updates the Glue catalog after the step completes")
glue_table = st.text_input("Glue table", value="user_events")
aws_region = st.text_input("AWS region", value="us-east-1")
st.divider()
st.subheader("π₯οΈ EMR Cluster")
emr_cluster_id = st.text_input("Cluster ID", value="j-XXXXXXXXXXXXX",
help="The ID of your running EMR cluster")
emr_script_s3 = st.text_input("Script S3 path", value="s3://my-bucket/scripts/job.py",
help="Upload glue/job.py to S3 and paste the URI here")
emr_py_files = st.text_input("--py-files S3 path (optional)",
value="s3://my-bucket/scripts/glue.zip",
help="Zip of the glue/ package. Run: zip -r glue.zip glue/")
emr_spark_conf = st.text_input("Extra Spark --conf (comma-separated)",
value="spark.executor.memory=8g,spark.executor.cores=4",
help="Override any spark config key=value pairs")
else:
output_dest = st.radio("Destination", ["Local file", "S3"])
output_path = st.text_input(
"Output path",
value="./output/user_events.parquet" if output_dest == "Local file" else "s3://my-bucket/user_events/",
)
if output_dest == "S3":
glue_db = st.text_input("Glue database (optional)", value="")
glue_table = st.text_input("Glue table", value="user_events")
aws_region = st.text_input("AWS region", value="us-east-1")
else:
glue_db = glue_table = aws_region = None
emr_cluster_id = emr_script_s3 = emr_py_files = emr_spark_conf = None
# ββ Main area βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
st.title("Data Quality Issue Simulator")
st.caption("103 issues across 9 categories Β· Parquet output Β· runs locally or on AWS EMR")
page_sim, page_report = st.tabs(["π§ͺ Simulator", "π Executive Report"])
# ββ Executive Report page βββββββββββββββββββββββββββββββββββββββββββββββββββββ
with page_report:
runs = _load_runs()
total_runs = len(runs)
r1, r2, r3, r4 = st.columns(4)
r1.metric("Total runs logged", total_runs)
if runs:
from collections import Counter
all_applied = [iid for r in runs for iid in r.get("issues_applied", [])]
sev_critical = sum(r.get("issues_by_severity", {}).get("critical", 0) for r in runs)
avg_rows = int(sum(r.get("rows_generated", 0) for r in runs) / total_runs)
r2.metric("Avg rows/run", f"{avg_rows:,}")
r3.metric("Critical issues (total)", sev_critical)
r4.metric("Unique issues used", len(set(all_applied)))
st.divider()
# Run history table
with st.expander("π Run history", expanded=False):
hist = [{
"Run ID": r["run_id"],
"Date": r["timestamp"][:19],
"Mode": r.get("execution_mode", "local"),
"Rows": f'{r.get("rows_generated",0):,}',
"Applied": len(r.get("issues_applied", [])),
"Skipped": len(r.get("issues_skipped", [])),
} for r in reversed(runs)]
st.dataframe(hist, use_container_width=True)
st.divider()
rep_col1, rep_col2 = st.columns([2, 8])
with rep_col1:
gen_report_btn = st.button("π€ Generate Report with Claude", type="primary",
use_container_width=True,
disabled=not bedrock_enabled)
if gen_report_btn:
with st.spinner("Claude is writing the executive report..."):
try:
report_md = ai_executive_report(
runs,
bedrock_model or anthropic_model,
bedrock_region,
)
st.session_state["exec_report_md"] = report_md
except Exception as e:
st.error(f"Error: {e}")
if "exec_report_md" in st.session_state:
st.markdown(st.session_state["exec_report_md"])
st.divider()
dl1, dl2, _ = st.columns([2, 2, 6])
with dl1:
st.download_button(
"β¬οΈ Download PDF",
data=_render_pdf(st.session_state["exec_report_md"], total_runs),
file_name="dq_executive_report.pdf",
mime="application/pdf",
use_container_width=True,
)
with dl2:
st.download_button(
"β¬οΈ Download Markdown",
data=st.session_state["exec_report_md"],
file_name="dq_executive_report.md",
mime="text/markdown",
use_container_width=True,
)
else:
st.info("No simulation runs yet. Run at least one simulation in the **π§ͺ Simulator** tab to generate a report.")
# ββ Simulator page ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
with page_sim:
# ββ Issue selector ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# ββ AI Issue Configurator βββββββββββββββββββββββββββββββββββββββββββββββββββββ
if bedrock_enabled:
with st.expander("π€ AI Issue Configurator β describe your scenario in plain English", expanded=False):
scenario_input = st.text_area(
"Describe the data quality problems you want to simulate",
placeholder=(
"e.g. 'Our pipeline sometimes receives duplicate events from mobile SDKs, "
"occasional bot traffic spikes, and purchases with missing product IDs. "
"We also see timestamp drift where events arrive out of order.'"
),
height=100,
)
ai_col1, ai_col2 = st.columns([2, 8])
with ai_col1:
ai_configure = st.button("π€ Configure with AI", type="secondary",
disabled=not scenario_input.strip(),
use_container_width=True)
if ai_configure and scenario_input.strip():
with st.spinner("Claude is selecting issues..."):
try:
selections = ai_select_issues(scenario_input, bedrock_model or anthropic_model, bedrock_region)
# Apply selections to session state
ai_ids = {s["id"] for s in selections if s["id"] in REGISTRY}
st.session_state.selected = ai_ids
# Store AI-suggested rates
st.session_state.ai_rates = {
s["id"]: s["rate"] for s in selections if s["id"] in REGISTRY
}
st.success(f"Claude selected **{len(ai_ids)} issues**. Scroll down to review and adjust.")
st.rerun()
except Exception as e:
st.error(f"Bedrock error: {e}")
# ββ AI Issue Creator ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
if bedrock_enabled:
with st.expander("β¨ AI Issue Creator β generate a brand new issue from a description", expanded=False):
new_issue_desc = st.text_area(
"Describe the new data quality issue",
placeholder="e.g. 'Create an issue where bot traffic is created by 10 unique bots, each generating 100 events per minute'",
height=80,
key="new_issue_desc",
)
create_col1, create_col2 = st.columns([2, 8])
with create_col1:
create_btn = st.button("β¨ Generate Issue", type="secondary",
disabled=not new_issue_desc.strip(),
use_container_width=True)
if create_btn and new_issue_desc.strip():
with st.spinner("Claude is writing the issue code..."):
try:
generated_code = ai_create_issue(new_issue_desc, bedrock_model or anthropic_model, bedrock_region)
st.session_state["pending_issue_code"] = generated_code
st.session_state["pending_issue_desc"] = new_issue_desc
except Exception as e:
st.error(f"Error: {e}")
if "pending_issue_code" in st.session_state:
st.markdown("**Generated code β review before adding:**")
edited_code = st.text_area("Edit if needed", value=st.session_state["pending_issue_code"],
height=300, key="edited_issue_code")
confirm_col1, confirm_col2, _ = st.columns([2, 2, 6])
with confirm_col1:
if st.button("β
Add to Simulator", type="primary", use_container_width=True):
try:
# Validate the code compiles
compile(edited_code, "<ai_issue>", "exec")
# Append to custom.py
custom_path = os.path.join(os.path.dirname(__file__),
"simulator", "issues", "custom.py")
with open(custom_path, "r") as f:
current = f.read()
# Strip the ISSUES = [] placeholder on first write
if current.strip().endswith("ISSUES = []"):
current = current.replace("ISSUES = []", "")
# Extract just the function + Issue(...) block, append to file
new_content = current.rstrip() + "\n\n" + edited_code.strip() + "\n"
# Merge ISSUES lists β replace last ISSUES = [...] with combined
with open(custom_path, "w") as f:
f.write(new_content)
# Reload the module and registry
import importlib
import simulator.issues.custom as custom_mod
import simulator.issues.registry as reg_mod
importlib.reload(custom_mod)
importlib.reload(reg_mod)
# Re-import updated REGISTRY into app scope
from simulator.issues.registry import REGISTRY as _NEW_REG
REGISTRY.clear()
REGISTRY.update(_NEW_REG)
del st.session_state["pending_issue_code"]
del st.session_state["pending_issue_desc"]
st.success("Issue added! It will appear in the list below.")
st.rerun()
except SyntaxError as e:
st.error(f"Syntax error in generated code: {e}")
except Exception as e:
st.error(f"Failed to add issue: {e}")
with confirm_col2:
if st.button("ποΈ Discard", use_container_width=True):
del st.session_state["pending_issue_code"]
st.rerun()
st.subheader("Select Issues to Inject")
# Global filter row
col_sev, col_cat, col_search, col_all, col_none = st.columns([2, 2, 3, 1, 1])
with col_sev:
sev_filter = st.selectbox("Severity filter", ["All", "critical", "high", "medium", "low"])
with col_cat:
cat_filter = st.selectbox("Category filter", ["All"] + [CATEGORY_LABELS[c] for c in CATEGORIES])
with col_search:
search = st.text_input("Search issues", placeholder="e.g. null, duplicate, timestamp ...")
with col_all:
st.write("")
select_all = st.button("β
All", use_container_width=True)
with col_none:
st.write("")
select_none = st.button("β None", use_container_width=True)
# Build filtered issue list
all_issues = list(REGISTRY.values())
filtered = all_issues
if sev_filter != "All":
filtered = [i for i in filtered if i.severity == sev_filter]
if cat_filter != "All":
cat_key = cat_filter.lower().replace(" ", "_")
filtered = [i for i in filtered if i.category == cat_key]
if search:
q = search.lower()
filtered = [i for i in filtered if q in i.id or q in i.description.lower()]
# Track selected issues in session state
if "selected" not in st.session_state:
st.session_state.selected = {i.id for i in all_issues}
if "ai_rates" not in st.session_state:
st.session_state.ai_rates = {}
if select_all:
st.session_state.selected = {i.id for i in filtered} | st.session_state.selected
if select_none:
st.session_state.selected -= {i.id for i in filtered}
# Render issues grouped by category
filtered_by_cat = {}
for issue in filtered:
filtered_by_cat.setdefault(issue.category, []).append(issue)
selected_ids = set(st.session_state.selected)
rate_overrides = {}
for cat in sorted(filtered_by_cat.keys()):
cat_issues = filtered_by_cat[cat]
cat_label = CATEGORY_LABELS[cat]
n_selected = sum(1 for i in cat_issues if i.id in selected_ids)
with st.expander(f"**{cat_label}** β {n_selected}/{len(cat_issues)} selected", expanded=True):
# Category-level select all / none
c1, c2, _ = st.columns([1, 1, 8])
with c1:
if st.button("All", key=f"all_{cat}", use_container_width=True):
for i in cat_issues:
st.session_state.selected.add(i.id)
st.rerun()
with c2:
if st.button("None", key=f"none_{cat}", use_container_width=True):
for i in cat_issues:
st.session_state.selected.discard(i.id)
st.rerun()
for issue in cat_issues:
cols = st.columns([0.5, 3.2, 1.5, 1.5, 2, 0.8])
with cols[0]:
checked = st.checkbox(
"Select", value=(issue.id in st.session_state.selected),
key=f"cb_{issue.id}",
label_visibility="collapsed",
)
if checked:
st.session_state.selected.add(issue.id)
else:
st.session_state.selected.discard(issue.id)
with cols[1]:
ai_tag = " π€" if issue.id in st.session_state.ai_rates else ""
st.markdown(f"`{issue.id}`{ai_tag} \n<small>{issue.description}</small>",
unsafe_allow_html=True)
with cols[2]:
st.markdown(severity_badge(issue.severity), unsafe_allow_html=True)
with cols[3]:
st.caption(f"Default rate: {issue.default_rate:.1%}")
with cols[4]:
if rate_mode == "Per-issue defaults":
ai_rate = st.session_state.ai_rates.get(issue.id, issue.default_rate)
custom = st.slider(
"rate", 0.001, 0.50,
float(ai_rate),
step=0.001, format="%.3f",
key=f"rate_{issue.id}",
label_visibility="collapsed",
disabled=(issue.id not in st.session_state.selected),
)
rate_overrides[issue.id] = custom
else:
st.caption("β global rate")
with cols[5]:
if bedrock_enabled and st.button("π¬", key=f"explain_{issue.id}",
help="Ask Claude to explain this issue"):
with st.spinner("Asking Claude..."):
try:
explanation = ai_explain_issue(issue.id, bedrock_model or anthropic_model, bedrock_region)
st.session_state[f"explain_text_{issue.id}"] = explanation
except Exception as e:
st.session_state[f"explain_text_{issue.id}"] = f"Error: {e}"
# Show explanation inline if available
if f"explain_text_{issue.id}" in st.session_state:
st.info(f"π¬ **{issue.name}:** {st.session_state[f'explain_text_{issue.id}']}")
# ββ Summary bar ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
st.divider()
active = [i for i in all_issues if i.id in st.session_state.selected]
by_sev = {}
for i in active:
by_sev[i.severity] = by_sev.get(i.severity, 0) + 1
summary_cols = st.columns(6)
with summary_cols[0]:
st.metric("Issues selected", len(active))
with summary_cols[1]:
st.metric("π£ Critical", by_sev.get("critical", 0))
with summary_cols[2]:
st.metric("π΄ High", by_sev.get("high", 0))
with summary_cols[3]:
st.metric("π‘ Medium", by_sev.get("medium", 0))
with summary_cols[4]:
st.metric("π’ Low", by_sev.get("low", 0))
with summary_cols[5]:
st.metric("Base rows", f"{n_rows:,}")
# ββ Generate button βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
st.divider()
run_col, _ = st.columns([2, 8])
with run_col:
btn_label = "βοΈ Submit Glue Job" if is_glue else "π Generate & Inject"
run = st.button(btn_label, type="primary", use_container_width=True,
disabled=(len(active) == 0))
if run and is_glue:
_run_emr_job(
issue_ids=[i.id for i in active],
n_rows=int(n_rows),
rate_override=global_rate,
output_path=output_path,
glue_database=glue_db,
glue_table=glue_table,
region=aws_region,
cluster_id=emr_cluster_id,
script_s3=emr_script_s3,
py_files_s3=emr_py_files or None,
n_partitions=n_partitions,
seed=int(seed),
extra_spark_conf=emr_spark_conf or "",
)
if run and not is_glue:
progress = st.progress(0, text="Generating clean data...")
log_area = st.empty()
logs = []
# 1. Generate
with st.spinner(f"Generating {n_rows:,} rows..."):
df = generator.generate(n_rows, seed=int(seed))
progress.progress(20, text=f"Generated {n_rows:,} clean rows")
# 2. Inject
issue_ids = [i.id for i in active]
# Build per-issue rates if in per-issue mode
if rate_mode == "Per-issue defaults":
# Temporarily patch each issue's default_rate
originals = {}
for iid in issue_ids:
originals[iid] = REGISTRY[iid].default_rate
REGISTRY[iid].default_rate = rate_overrides.get(iid, REGISTRY[iid].default_rate)
rate_arg = None
else:
originals = {}
rate_arg = global_rate
applied = []
skipped = []
rng = np.random.default_rng(int(seed))
progress.progress(25, text="Injecting issues...")
for idx, iid in enumerate(issue_ids):
issue = REGISTRY[iid]
rate = rate_arg if rate_arg is not None else issue.default_rate
try:
df = issue.inject(df, rate, rng)
applied.append(iid)
logs.append(f"β
{iid}")
except Exception as e:
skipped.append(iid)
logs.append(f"β οΈ {iid} β skipped ({e})")
pct = 25 + int(70 * (idx + 1) / len(issue_ids))
progress.progress(pct, text=f"Injecting: {iid}")
log_area.text("\n".join(logs[-20:]))
# Restore original rates
for iid, orig in originals.items():
REGISTRY[iid].default_rate = orig
# 3. Write
progress.progress(96, text="Writing parquet...")
try:
out = writer.write(
df,
output=output_path,
glue_database=glue_db or None,
glue_table=glue_table or "user_events",
region=aws_region or "us-east-1",
)
write_ok = True
write_msg = f"Written to `{out}`"
except Exception as e:
write_ok = False
write_msg = f"Write failed: {e}"
progress.progress(100, text="Done!")
log_area.empty()
# ββ Save run to log βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
_save_run(_build_run_record(applied, skipped, df, output_path, exec_mode))
# ββ Results βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
st.success(f"Done! **{len(applied)}** issues injected Β· **{df.shape[0]:,}** rows Β· **{df.shape[1]}** columns")
if skipped:
st.warning(f"{len(skipped)} issues skipped: {', '.join(skipped)}")
if write_ok:
st.info(write_msg)
else:
st.error(write_msg)
# Stats
st.subheader("Results")
r1, r2, r3, r4 = st.columns(4)
r1.metric("Total rows", f"{df.shape[0]:,}", delta=f"+{df.shape[0]-n_rows:,} from injection")
r2.metric("Columns", df.shape[1])
r3.metric("Issues applied", len(applied))
r4.metric("Issues skipped", len(skipped))
# Tabs: preview, nulls, types, log
tab_preview, tab_nulls, tab_types, tab_log, tab_report, tab_download = st.tabs(
["π Preview", "π³οΈ Null Analysis", "π¬ Type Analysis", "π Injection Log", "π€ AI DQ Report", "β¬οΈ Download"]
)
with tab_preview:
st.caption(f"First 200 rows of {df.shape[0]:,} total")
st.dataframe(df.head(200), use_container_width=True, height=400)
with tab_nulls:
null_counts = df.isnull().sum().reset_index()
null_counts.columns = ["column", "null_count"]
null_counts["null_%"] = (null_counts["null_count"] / len(df) * 100).round(2)
null_counts = null_counts[null_counts["null_count"] > 0].sort_values("null_count", ascending=False)
if null_counts.empty:
st.info("No null values found.")
else:
st.bar_chart(null_counts.set_index("column")["null_%"], use_container_width=True)
st.dataframe(null_counts, use_container_width=True)
with tab_types:
type_df = pd.DataFrame({
"column": df.columns,
"dtype": [str(df[c].dtype) for c in df.columns],
"non_null": [df[c].count() for c in df.columns],
"unique": [df[c].nunique() for c in df.columns],
"sample_value": [str(df[c].dropna().iloc[0]) if df[c].count() > 0 else "N/A" for c in df.columns],
})
st.dataframe(type_df, use_container_width=True)
with tab_log:
st.code("\n".join(logs))
with tab_report:
if not bedrock_enabled:
st.info("Enable AI features in the sidebar to generate a DQ report.")
else:
if st.button("π€ Generate DQ Report with Claude", type="secondary"):
null_pct = {
col: round(df[col].isnull().sum() / len(df) * 100, 1)
for col in df.columns
}
df_stats = {
"rows": len(df),
"cols": df.shape[1],
"null_pct": null_pct,
}
with st.spinner("Claude is writing your DQ report..."):
try:
report = ai_dq_report(applied, df_stats, bedrock_model or anthropic_model, bedrock_region)
st.session_state["last_dq_report"] = report
except Exception as e:
st.error(f"Bedrock error: {e}")
if "last_dq_report" in st.session_state:
st.markdown(st.session_state["last_dq_report"])
st.download_button(
"β¬οΈ Download Report (.md)",
data=st.session_state["last_dq_report"],
file_name="dq_report.md",
mime="text/markdown",
)
with tab_download:
buf = io.BytesIO()
df_str = df.astype(str)
try:
import pyarrow as pa
import pyarrow.parquet as pq
safe = {}
for col in df.columns:
try:
safe[col] = pa.array(df[col].tolist())
except Exception:
safe[col] = pa.array(df[col].astype(str).tolist(), type=pa.string())
table = pa.table(safe)
pq.write_table(table, buf)
buf.seek(0)
st.download_button(
label="β¬οΈ Download as Parquet",
data=buf,
file_name="user_events_dq.parquet",
mime="application/octet-stream",
use_container_width=True,
)
except Exception as e:
st.error(f"Could not prepare download: {e}")
csv_buf = io.StringIO()
df.astype(str).to_csv(csv_buf, index=False)
st.download_button(
label="β¬οΈ Download as CSV",
data=csv_buf.getvalue(),
file_name="user_events_dq.csv",
mime="text/csv",
use_container_width=True,
)
"""Streamlit UI for the Data Quality Issue Simulator."""
import io
import sys
import os
import time
# Ensure the simulator package is importable when run from this directory
sys.path.insert(0, os.path.dirname(__file__))
import streamlit as st
import pandas as pd
import numpy as np
from simulator import generator, injector, writer
from simulator.issues import REGISTRY, list_issues
# ββ Page config ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
st.set_page_config(
page_title="DQ Issue Simulator",
page_icon="π§ͺ",
layout="wide",
initial_sidebar_state="expanded",
)
# ββ Helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
SEVERITY_COLOR = {
"low": "#6c757d",
"medium": "#fd7e14",
"high": "#dc3545",
"critical": "#6f42c1",
}
SEVERITY_EMOJI = {"low": "π’", "medium": "π‘", "high": "π΄", "critical": "π£"}
CATEGORIES = sorted({i.category for i in REGISTRY.values()})
CATEGORY_LABELS = {c: c.replace("_", " ").title() for c in CATEGORIES}
def severity_badge(sev: str) -> str:
color = SEVERITY_COLOR.get(sev, "#888")
return f'<span style="background:{color};color:#fff;padding:2px 8px;border-radius:4px;font-size:0.75rem;font-weight:600">{sev.upper()}</span>'
# ββ Bedrock / Claude helpers ββββββββββββββββββββββββββββββββββββββββββββββββββ
def _call_claude(prompt: str, model_id: str, region: str, max_tokens: int = 4096) -> str:
"""Call Claude via Anthropic API or Bedrock depending on selected provider."""
if ai_provider == "Anthropic API":
try:
import anthropic
except ImportError:
raise RuntimeError("Run: pip install anthropic")
if not anthropic_api_key:
raise RuntimeError("Enter your Anthropic API key in the sidebar.")
client = anthropic.Anthropic(api_key=anthropic_api_key)
msg = client.messages.create(
model=anthropic_model,
max_tokens=max_tokens,
messages=[{"role": "user", "content": prompt}],
)
return msg.content[0].text
else: # AWS Bedrock
import boto3, json
kwargs = {"region_name": region}
if aws_access_key and aws_secret_key:
kwargs["aws_access_key_id"] = aws_access_key
kwargs["aws_secret_access_key"] = aws_secret_key
if aws_session_token:
kwargs["aws_session_token"] = aws_session_token
client = boto3.client("bedrock-runtime", **kwargs)
body = json.dumps({
"anthropic_version": "bedrock-2023-05-31",
"max_tokens": max_tokens,
"messages": [{"role": "user", "content": prompt}],
})
resp = client.invoke_model(modelId=model_id, body=body)
return json.loads(resp["body"].read())["content"][0]["text"]
def _issues_catalogue() -> str:
"""Compact text listing of all issues for prompt context."""
lines = []
for i in REGISTRY.values():
lines.append(f"{i.id} | {i.category} | {i.severity} | {i.description}")
return "\n".join(lines)
def ai_select_issues(scenario: str, model_id: str, region: str) -> list:
"""
Ask Claude to pick relevant issues and rates for a plain-English scenario.
Returns list of {"id": str, "rate": float}.
"""
import json
prompt = f"""You are a data quality expert. A user wants to simulate data quality issues
in a user events dataset (clicks, purchases, logins, etc.) stored in AWS S3 / Glue.
Available issues (id | category | severity | description):
{_issues_catalogue()}
User scenario: